# HG changeset patch # User mzhuang # Date 1632430848 0 # Node ID 4d2a2268a17afd29dc581ed9b7a24276f5c6b5e5 "planemo upload" diff -r 000000000000 -r 4d2a2268a17a chess_preprocessor.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/chess_preprocessor.xml Thu Sep 23 21:00:48 2021 +0000 @@ -0,0 +1,135 @@ + + for pre-processing raw data + + hexrd + + > '$output'; + + #if $input_source_conditional.input_source == "input_source_history" + #set $input_type = $input_source_conditional.input_type_conditional.input_type + #if $input_type == "single" + #set $input_data = $input_source_conditional.input_type_conditional.fastq_input1 + #elif $input_type == "list_collection" + #set $input_data = $input_source_conditional.input_type_conditional.fastq_input + #end if# + + #set $var = "" + #for $f in $input_data# + #set $var += "${f}\n" + #end for# + #set files = '\\n'.join([str($file) for $file in $input_data]) + ##for $f in $input_data# + #if $output_as_list == "Yes" + ##printf "$files" >> '$output_list_log' 2>&1; + printf "$files" | xargs -n 1 -P \${GALAXY_SLOTS} -I{} python '$__tool_directory__/preprocess_dexela_h5_galaxy.py' {} -o '$ome_start' -s '$start_frame' -n '$num_frames' -e '$ome_end' -t '$threshold' >> '$output_list_log' 2>&1; + #elif $output_as_list == "No" + printf "$files" | xargs -n 1 -P \${GALAXY_SLOTS} -I{} python '$__tool_directory__/preprocess_dexela_h5_galaxy.py' {} -o '$ome_start' -s '$start_frame' -n '$num_frames' -e '$ome_end' -t '$threshold' >> '$output' 2>&1; + #end if# + ##end for# + #elif $input_source_conditional.input_source == "input_source_linux" + #set $scans = [i for a in [(int(j[0]),int(j[-1])+1) for j in [x.split(':') for x in $input_source_conditional.scan_numbers.split(',')]] for i in range(a[0],a[1])] + ##set $scanlist = '\n'.join($scans) + + #for $n in $scans# + #if $output_as_list == "Yes" + echo '$input_source_conditional.base_dir' >> '$output_list_log'; + python '$__tool_directory__/preprocess_dexela_h5_filesystem.py' '$input_source_conditional.base_dir' '$input_source_conditional.expt_name' '$input_source_conditional.samp_name' '$n' -o '$ome_start' -s '$start_frame' -n '$num_frames' -e '$ome_end' -t '$threshold' >> '$output_list_log' 2>&1; + #elif $output_as_list == "No" + echo '$input_source_conditional.base_dir' >> '$output'; + python '__tool_directory__/preprocess_dexela_h5_filesystem.py' '$input_source_conditional.base_dir' '$input_source_conditional.expt_name' '$input_source_conditional.samp_name' '$n' -o '$ome_start' -s '$start_frame' -n '$num_frames' -e '$ome_end' -t '$threshold' >> '$output' 2>&1; + #end if# + #end for# + #end if# + conda deactivate + ]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + output_as_list == 'Yes' + + + + + + + + + output_as_list == 'Yes' + + + + output_as_list == 'No' + + + + + diff -r 000000000000 -r 4d2a2268a17a pp_dexela.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pp_dexela.py Thu Sep 23 21:00:48 2021 +0000 @@ -0,0 +1,153 @@ +from __future__ import print_function + +import time +import os +import numpy as np + +from hexrd import imageseries +from hexrd.imageseries.process import ProcessedImageSeries + + + + +class ProcessedDexelaIMS(ProcessedImageSeries): + + ADDROW = 'add-row' + ADDCOL = 'add-column' + + def __init__(self, imser, oplist, **kwargs): + super(ProcessedDexelaIMS, self).__init__(imser, oplist, **kwargs) + self.addop(self.ADDROW, self._addrow) + self.addop(self.ADDCOL, self._addcol) + + def _addrow(self, img, k): + """insert row into position k""" + shp = img.shape + pimg = np.insert(img, k, 0, axis=0) + if k==0: + pimg[0] = pimg[1] + elif k==shp[0]: + pimg[k] = pimg[k-1] + else: # in middle + pimg[k] = (pimg[k-1] + pimg[k+1])/2 + + return pimg + + def _addcol(self, img, k): + """insert row into position k""" + shp = img.shape + pimg = np.insert(img, k, 0, axis=1) + if k==0: + pimg[:,0] = pimg[:,1] + elif k==shp[0]: + pimg[:,k] = pimg[:,k-1] + else: # in middle + pimg[:,k] = (pimg[:,k-1] + pimg[:,k+1])/2 + + return pimg + + +DexelaIMS=ProcessedDexelaIMS + +class PP_Dexela(object): + """PP_Dexela""" + PROCFMT = 'frame-cache' + RAWFMT = 'hdf5' + RAWPATH = '/imageseries' + DARKPCTILE = 50 + + def __init__(self, + fname, omw, panel_opts, panel_id, + frame_start=0, raw_format='hdf5',dark=None): + """Constructor for PP_Dexela""" + self._panel_id = panel_id + self.fname = fname + self.omwedges = omw + self.panel_opts = panel_opts + self.frame_start = frame_start + self.use_frame_list = (self.frame_start > 0) + if raw_format.lower() == 'hdf5': + self.raw = imageseries.open( + self.fname, self.RAWFMT, path=self.RAWPATH + ) + else: + self.raw = imageseries.open(self.fname, raw_format.lower()) + self._dark = dark + + #print( + # 'On Init:\n\t%s, %d frames, %d omw, %d total' + # % (self.fname, self.nframes, self.omwedges.nframes, len(self.raw)) + #) + + @property + def panel_id(self): + return self._panel_id + + @property + def oplist(self): + return [('dark', self.dark)]+self.panel_opts + + @property + def framelist(self): + return range(self.frame_start, self.nframes + self.frame_start) + + # + # ============================== API + # + @property + def nframes(self): + return self.omwedges.nframes + + @property + def omegas(self): + return self.omwedges.omegas + + def processed(self): + kw = {} + if self.use_frame_list: + kw = dict(frame_list=self.framelist) + return DexelaIMS(self.raw, self.oplist, **kw) + + @property + def dark(self, nframes=100): + """build and return dark image""" + if self._dark is None: + usenframes = min(nframes, self.nframes) + print( + "building dark images using %s frames (may take a while)..." + % usenframes + ) + start = time.time() +# self._dark = imageseries.stats.percentile( +# self.raw, self.DARKPCTILE, nframes=usenframes +# ) + self._dark = imageseries.stats.median( + self.raw, nframes=usenframes + )#changed to median by DCP 11/18/17 + elapsed = (time.time() - start) + print( + "done building background (dark) image: " + + "elapsed time is %f seconds" % elapsed + ) + + return self._dark + + def save_processed(self, name, threshold, output_dir=None): + if output_dir is None: + output_dir = os.getcwd() + else: + os.mkdir(output_dir) + + # add omegas + pims = self.processed() + metad = pims.metadata + metad['omega'] = self.omegas + metad['panel_id'] = self.panel_id + cache = '%s-cachefile.npz' % name + imageseries.write(pims, "dummy", self.PROCFMT, + style="npz", + threshold=threshold, + cache_file=cache) + pass # end class + + diff -r 000000000000 -r 4d2a2268a17a preprocess_dexela_h5_filesystem.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/preprocess_dexela_h5_filesystem.py Thu Sep 23 21:00:48 2021 +0000 @@ -0,0 +1,140 @@ +import argparse +import glob +import os + +import numpy as np +import time +from datetime import datetime + +from hexrd.imageseries import omega +import pp_dexela +#from IPython import embed + +# ============================================================================= +# USER INPUT +# ============================================================================= + +# panel keys to MATCH INSTRUMENT FILE +panel_keys = ['FF1', 'FF2'] +panel_opts = dict.fromkeys(panel_keys) + +# !!!: hard coded options for each dexela for April 2017 +panel_opts['FF1'] = [('add-row', 1944), ('add-column', 1296),('flip', 'v'), ] +panel_opts['FF2'] = [('add-row', 1944), ('add-column', 1296),('flip', 'h'), ] + +# ==================== End Inputs (should not need to alter below this line) + + + +def process_dexelas(file_names, samp_name, scan_number, + ostart, ostep, num_frames, + panel_opts, threshold): + """ + wrapper for F2 dexela setup + """ + ostop = ostart + num_frames*ostep + omw = omega.OmegaWedges(num_frames) + omw.addwedge(ostart, ostop, num_frames) + + for file_name in file_names: + for key in panel_keys: + if key.lower() in file_name: + ppd = pp_dexela.PP_Dexela( + file_name, + omw, + panel_opts[key], + panel_id=key, + frame_start=fstart) +# embed() +# ppd=add_missing_pixel_gap(ppd) + + output_name = samp_name + '_' + \ + str(scan_number) + '_' + \ + file_name.split('/')[-1].split('.')[0] + ppd.save_processed(output_name, threshold) + head_tail = os.path.split(file_name) + if head_tail[0] is not None: + os.system('cp -p ' + './' + output_name + '* ' + head_tail[0] + ' 2>/dev/null') + +if __name__ == '__main__': + # + # Run preprocessor + # + print("Running preprocessor....") + parser = argparse.ArgumentParser( + description="pre-process double Dexela images from F2") + + parser.add_argument('base_dir', + help="raw data path on chess daq", type=str) + parser.add_argument('expt_name', + help="experiment name", type=str) + parser.add_argument('samp_name', + help="sample name", type=str) + parser.add_argument('scan_number', + help="ff scan number", type=int) + + parser.add_argument('-n', '--num-frames', + help="number of frames to read", + type=int, default=1441) + parser.add_argument('-s', '--start-frame', + help="index of first data frame", + type=int, default=4) + parser.add_argument('-t', '--threshold', + help="threshold for frame caches", + type=int, default=50) + parser.add_argument('-o', '--ome-start', + help="start omega", + type=float, default=0.) +# parser.add_argument('-d', '--ome-delta', +# help="delta omega", +# type=float, default=0.2498265093684941) + + parser.add_argument('-e', '--ome-end', + help="end omega", + type=float, default=360.) + + + args = parser.parse_args() + + # strip args + data_dir = args.base_dir + expt_name = args.expt_name + samp_name = args.samp_name + scan_number = args.scan_number + num_frames = args.num_frames + fstart = args.start_frame + threshold = args.threshold + ostart = args.ome_start + oend = args.ome_end + ostep = (oend-ostart)/float(num_frames) + + print("== Running preprocessor .... processing ") + now = datetime.now() + print(now.strftime("%m/%d/%Y %H:%M:%S")) + + file_names = glob.glob( + os.path.join( + data_dir, + expt_name, + samp_name, + str(scan_number), + 'ff', + '*.h5') + ) + path = os.path.join(data_dir, expt_name, samp_name, str(scan_number), 'ff', '*.h5') + print("path = " + path) + for f in file_names: + print(f) + + check_files_exist = [os.path.exists(file_name) for file_name in file_names] + if not np.all(check_files_exist): + raise RuntimeError("files don't exist!") + + process_dexelas(file_names, samp_name, scan_number, + ostart, ostep, num_frames, + panel_opts, threshold) + now = datetime.now() + print(now.strftime("%m/%d/%Y %H:%M:%S")) + print("== End of processing " + " ==") + print(" ") + diff -r 000000000000 -r 4d2a2268a17a preprocess_dexela_h5_galaxy.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/preprocess_dexela_h5_galaxy.py Thu Sep 23 21:00:48 2021 +0000 @@ -0,0 +1,151 @@ +import argparse +import glob +import os + +import numpy as np + +import time +from datetime import datetime + +from hexrd.imageseries import omega +import pp_dexela +#from IPython import embed + +# ============================================================================= +# USER INPUT +# ============================================================================= + +# panel keys to MATCH INSTRUMENT FILE +panel_keys = ['FF1', 'FF2'] +panel_opts = dict.fromkeys(panel_keys) + +# !!!: hard coded options for each dexela for April 2017 +panel_opts['FF1'] = [('add-row', 1944), ('add-column', 1296),('flip', 'v'), ] +panel_opts['FF2'] = [('add-row', 1944), ('add-column', 1296),('flip', 'h'), ] + +# ==================== End Inputs (should not need to alter below this line) + + + +def process_dexelas(file_names, samp_name, scan_number, + ostart, ostep, num_frames, + panel_opts, threshold): + """ + wrapper for F2 dexela setup + """ + ostop = ostart + num_frames*ostep + omw = omega.OmegaWedges(num_frames) + omw.addwedge(ostart, ostop, num_frames) + + for file_name in file_names: + for key in panel_keys: + if key.lower() in file_name: + ppd = pp_dexela.PP_Dexela( + file_name, + omw, + panel_opts[key], + panel_id=key, + frame_start=fstart) +# embed() +# ppd=add_missing_pixel_gap(ppd) + + #output_name = samp_name + '_' + \ + # str(scan_number) + '_' + \ + output_name = file_name.split('/')[-4] + '_' + \ + file_name.split('/')[-3]+ '_' + \ + file_name.split('/')[-1].split('.')[0] + ppd.save_processed(output_name, threshold) + head_tail = os.path.split(file_name) + if head_tail[0] is not None: + os.system('cp -p ' + './' + output_name + '* ' + head_tail[0] + ' 2>/dev/null') +if __name__ == '__main__': + # + # Run preprocessor + # + + parser = argparse.ArgumentParser( + description="pre-process double Dexela images from F2") + + parser.add_argument('file_name', + help="raw data path on chess daq", type=str) + + #parser.add_argument('base_dir', + # help="raw data path on chess daq", type=str) + #parser.add_argument('expt_name', + # help="experiment name", type=str) + #parser.add_argument('samp_name', + # help="sample name", type=str) + #parser.add_argument('scan_number', + # help="ff scan number", type=int) + + parser.add_argument('-n', '--num-frames', + help="number of frames to read", + type=int, default=1441) + parser.add_argument('-s', '--start-frame', + help="index of first data frame", + type=int, default=4) + parser.add_argument('-t', '--threshold', + help="threshold for frame caches", + type=int, default=50) + parser.add_argument('-o', '--ome-start', + help="start omega", + type=float, default=0.) +# parser.add_argument('-d', '--ome-delta', +# help="delta omega", +# type=float, default=0.2498265093684941) + + parser.add_argument('-e', '--ome-end', + help="end omega", + type=float, default=360.) + + + args = parser.parse_args() + + print(args.file_name) + # strip args + file_name = args.file_name + #data_dir = args.base_dir + #expt_name = args.expt_name + #samp_name = args.samp_name + #scan_number = args.scan_number + samp_name = file_name.split('/')[-4] + scan_number = file_name.split('/')[-3] + num_frames = args.num_frames + fstart = args.start_frame + threshold = args.threshold + ostart = args.ome_start + oend = args.ome_end + ostep = (oend-ostart)/float(num_frames) + + print("== Running preprocessor .... processing " + file_name + " ==") + now = datetime.now() + print(now.strftime("%m/%d/%Y %H:%M:%S")) + + #file_names = glob.glob( + # os.path.join( + # data_dir, + # expt_name, + # samp_name, + # str(scan_number), + # 'ff', + # '*.h5') + #) + file_names = [file_name] + #path = os.path.join(data_dir, expt_name, samp_name, str(scan_number), 'ff', '*.h5') + #print("Preprocessor: after getting file_names ...." + path) + for f in file_names: + print(f) + + check_files_exist = [os.path.exists(file_name) for file_name in file_names] + if not np.all(check_files_exist): + raise RuntimeError("files don't exist!") + + process_dexelas(file_names, samp_name, scan_number, + ostart, ostep, num_frames, + panel_opts, threshold) + now = datetime.now() + print(now.strftime("%m/%d/%Y %H:%M:%S")) + print("== End of processing " + file_name + " ==") + print(" ") + print(" ") +