Galaxy |

Changeset 2:d150ac3d853d (2019-08-28)

Previous changeset 1:750c1684d47c (2019-02-18) Next changeset 3:2ddc36385d7a (2019-09-10)

Commit message:
"planemo upload for repository https://github.com/bgruening/galaxytools/tree/master/tools/text_processing/split_file_to_collection commit 06ffe450bafa280eee8a4331c9cfc9e1ece7c522"

modified:
split_file_to_collection.py
split_file_to_collection.xml

added:
test-data/3_molecules.sdf
test-data/mol_0.sdf
test-data/mol_1.sdf
test-data/mol_2.sdf

diff -r 750c1684d47c -r d150ac3d853d split_file_to_collection.py
--- a/split_file_to_collection.py Mon Feb 18 15:20:56 2019 -0500
+++ b/split_file_to_collection.py Wed Aug 28 10:55:25 2019 -0400

[

@@ -15,6 +15,7 @@
FILETYPES = {'fasta': '^>',
              'fastq': '^@',
              'tabular': '^.*',
+             'txt': '^.*',
              'mgf': '^BEGIN IONS'}

@@ -37,6 +38,8 @@

     ftype = args["ftype"]

+    assert ftype != "generic" or args["generic_re"] != None, "--generic_re needs to be given for generic input"
+
     if args["ftype"] == "tabular" and args["by"] == "col":
         args["match"] = replace_mapped_chars(args["match"])
         args["sub"] = replace_mapped_chars(args["sub"])
@@ -56,7 +59,8 @@
     parser.add_argument('--file_ext', '-e', help="If not splitting by column," +
                                                  " the extension of the new files (without a period)")
     parser.add_argument('--ftype', '-f', help="The type of the file to split", required = True,
-        choices=["mgf", "fastq", "fasta", "tabular"])
+        choices=["mgf", "fastq", "fasta", "tabular", "txt", "generic"])
+    parser.add_argument('--generic_re', '-g', help="Regular expression indicating the start of a new record (only for generic)", required = False)
     parser.add_argument('--by', '-b', help="Split by line or by column (tabular only)",
         default = "row", choices = ["col", "row"])
     parser.add_argument('--top', '-t', type=int, default=0, help="Number of header lines to carry over to new files. " +
@@ -96,7 +100,7 @@

def split_by_record(args, in_file, out_dir, top, ftype):
     # get record separator for given filetype
-    sep = re.compile(FILETYPES[ftype])
+    sep = re.compile(FILETYPES.get(ftype, args["generic_re"]))

     numnew = args["numnew"]

diff -r 750c1684d47c -r d150ac3d853d split_file_to_collection.xml
--- a/split_file_to_collection.xml Mon Feb 18 15:20:56 2019 -0500
+++ b/split_file_to_collection.xml Wed Aug 28 10:55:25 2019 -0400

[

b'@@ -1,6 +1,19 @@\n-<tool id="split_file_to_collection" name="Split file" version="0.1.1">\n+<tool id="split_file_to_collection" name="Split file" version="0.2.0">\n <description>to dataset collection</description>\n <macros>\n+ <xml name="regex_sanitizer">\n+ <sanitizer>\n+ <valid>\n+ <add preset="string.printable"/>\n+ <remove value="\" />\n+ <remove value="'" />\n+ </valid>\n+ <mapping initial="none">\n+ <add source="\" target="__backslash__" />\n+ <add source="'" target="__sq__"/>\n+ </mapping>\n+ </sanitizer>\n+ </xml>\n <xml name="numnew_fname">\n <param name="numnew" type="integer" label="Number of new files" min="1" value="1"/>\n <param name="newfilenames" type="text" label="Base name for new files in collection"\n@@ -48,6 +61,9 @@\n #end if\n #end if\n #else\n+ #if $split_parms.select_ftype == "generic"\n+ --generic_re \'$split_parms.generic_regex\'\n+ #end if\n --numnew \'$split_parms.numnew\'\n #if $split_parms.select_allocate.allocate == "random":\n --rand\n@@ -63,7 +79,11 @@\n #end if\n #if $split_parms.select_ftype != "tabular":\n --file_names \'$split_parms.newfilenames\'\n- --file_ext \'$split_parms.select_ftype\'\n+ #if $split_parms.select_ftype == "generic"\n+ --file_ext \'$split_parms.input.ext\'\n+ #else\n+ --file_ext \'$split_parms.select_ftype\'\n+ #end if\n #end if\n ]]></command>\n <inputs>\n@@ -73,6 +93,8 @@\n <option value="fastq">FASTQ</option>\n <option value="tabular">Tabular</option>\n <option value="fasta">FASTA</option>\n+ <option value="txt">Text files</option>\n+ <option value="generic">Generic</option>\n </param>\n <when value="tabular">\n <param name="input" type="data" format="tabular" label="Tabular file to split"/>\n@@ -85,30 +107,10 @@\n <when value="col">\n <param name="id_col" type="data_column" label="Column to split on" data_ref="input"/>\n <param name="match_regex" type="text" label="Regex to match contents of id column" value="(.*)">\n-\t \t\t <sanitizer>\n- \t\t <valid>\n- \t\t <add preset="string.printable"/>\n- \t\t <remove value="\" />\n- \t\t <remove value="'" />\n- \t\t\t </valid>\n- \t\t <mapping initial="none">\n- \t\t <add source="\" target="__backslash__" />\n- \t\t <add source="'" target="__sq__"/>\n- \t\t </mapping>\n-\t\t\t </sanitizer>\n-\t\t </param>\n+ <expand macro="regex_sanitizer"/>\n+ </param>\n <param name="sub_regex" type="text" label="Pattern to replace match with" value="\\1">\n- <sanitizer>\n- <valid>\n- <add preset="string.printable"/>\n- <remove value="\" />\n- <remove value="'" />\n- </valid>\n- <mapping initial="none">\n- <add source="\" target="__backslash__" />\n- <add source="'" target="__sq__"/>\n- </mapping>\n-\t\t\t </sanitizer>\n+ <expand macro="re'..b'gex" value="^>.*"/>\n+ <param name="numnew" value="2"/>\n+ <param name="newfilenames" value="rand"/>\n+ <param name="allocate" value="random"/>\n+ <param name="seed" value="1010"/> \n+ <output_collection name="list_output_generic" type="list">\n+ <element name="rand_0" file="rand_0.fasta" ftype="fasta"/>\n+ <element name="rand_1" file="rand_1.fasta" ftype="fasta"/>\n+ </output_collection>\n+ </test>\n+ <test>\n+ <param name="input" value="3_molecules.sdf" ftype="sdf"/>\n+ <param name="select_ftype" value="generic"/>\n+ <param name="generic_regex" value="^\\$\\$\\$\\$.*"/>\n+ <param name="numnew" value="1000"/>\n+ <param name="newfilenames" value="mol"/>\n+ <param name="allocate" value="batch"/>\n+ <output_collection name="list_output_generic" type="list">\n+ <element name="mol_0" file="mol_0.sdf" ftype="sdf"/>\n+ <element name="mol_1" file="mol_1.sdf" ftype="sdf"/>\n+ <element name="mol_2" file="mol_2.sdf" ftype="sdf"/>\n+ </output_collection>\n+ </test>\n </tests>\n <help><![CDATA[\n **Split file into a dataset collection**\n \n-This tool can split five types of files into a separate files within a dataset collection: MGF, FASTA, FASTQ, and tabular.\n+This tool splits a data sets consisting of records into multiple data sets within a collection. \n+A record can be for instance simply a line, a FASTA sequence (header + sequence), a FASTQ sequence\n+(headers + sequence + qualities), etc. The important property is that the begin of a new record\n+can be speciefied by a regular expression, e.g. ".*" for lines, ">.*" for FASTA, or "@.*" for FASTQ. \n+The tool has presets for text, tabular data sets (which are split by line), FASTA, FASTQ, and MGF. \n+For other data types the text delimiting records can be specified manually using the generic splitter. \n+\n+If splitting by line (or by some other item, like a FASTA entry or an MGF record, the splitting can be either done alternating, in original record order, or at random. \n+\n+If t records are to be distributed to n new data sets, then the i-th record goes to data set\n+\n+* floor(i / t * n) (for batch), \n+* i % n (for alternating), or\n+* a random data set\n+\n+For instance, t=5 records are distributed as follows on n=2 data sets\n+\n+= === === ====\n+i bat alt rand\n+= === === ====\n+0 0 0 0\n+1 0 1 1\n+2 0 0 1\n+3 1 1 0\n+4 1 0 0\n+= === === ====\n+\n+If the five records are distributed on n=3 data sets:\n+\n+= === === ====\n+i bat alt rand\n+= === === ====\n+0 0 0 0\n+1 0 1 1\n+2 1 2 2\n+3 1 0 0\n+4 2 1 1\n+= === === ====\n+\n+Note that there are no guarantees when splitting at random that every result file will be non-empty, so downstream tools should be able to gracefully handle empty files. \n+\n If a tabular file is used as input, you may choose to split by line or by column. If split by column, a new file is created for each unique value in the column.\n In addition, (Python) regular expressions may be used to transform the value in the column to a new value. Caution should be used with this feature, as it could transform all values to the same value, or other unexpected behavior.\n The default regular expression uses each value in the column without modifying it. \n-\n-If splitting by line (or by some other item, like a FASTA entry or an MGF section), the splitting can be either done sequentially or at random. \n-Note that there are no guarantees when splitting at random that every result file will be non-empty, so downstream tools should be able to gracefully handle empty files. \n-\n-**Note**\n-\n-Due to current limitations with dataset collections, a log file is produced when running this tool. It will usually be empty, but if the tool fails, any errors will be printed to the log file. \n ]]></help>\n <citations>\n <citation type="bibtex">\n'

diff -r 750c1684d47c -r d150ac3d853d test-data/3_molecules.sdf
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/3_molecules.sdf Wed Aug 28 10:55:25 2019 -0400

b'@@ -0,0 +1,350 @@\n+\n+ Chemfp\n+\n+ 21 21 0 0 0 0 0 0 0 0999 V2000\n+ 1.2333 0.5540 0.7792 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.6952 -2.7148 -0.7502 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ 0.7958 -2.1843 0.8685 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ 1.7813 0.8105 -1.4821 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.0857 0.6088 0.4403 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.7927 -0.5515 0.1244 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.7288 1.8464 0.4133 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.1426 -0.4741 -0.2184 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.0787 1.9238 0.0706 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.7855 0.7636 -0.2453 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.1409 -1.8536 0.1477 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ 2.1094 0.6715 -0.3113 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.5305 0.5996 0.1635 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.1851 2.7545 0.6593 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.7247 -1.3605 -0.4564 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.5797 2.8872 0.0506 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -3.8374 0.8238 -0.5090 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.7290 1.4184 0.8593 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 4.2045 0.6969 -0.6924 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.7105 -0.3659 0.6426 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.2555 -3.5916 -0.7337 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 1 5 1 0 0 0 0\n+ 1 12 1 0 0 0 0\n+ 2 11 1 0 0 0 0\n+ 2 21 1 0 0 0 0\n+ 3 11 2 0 0 0 0\n+ 4 12 2 0 0 0 0\n+ 5 6 1 0 0 0 0\n+ 5 7 2 0 0 0 0\n+ 6 8 2 0 0 0 0\n+ 6 11 1 0 0 0 0\n+ 7 9 1 0 0 0 0\n+ 7 14 1 0 0 0 0\n+ 8 10 1 0 0 0 0\n+ 8 15 1 0 0 0 0\n+ 9 10 2 0 0 0 0\n+ 9 16 1 0 0 0 0\n+ 10 17 1 0 0 0 0\n+ 12 13 1 0 0 0 0\n+ 13 18 1 0 0 0 0\n+ 13 19 1 0 0 0 0\n+ 13 20 1 0 0 0 0\n+M END\n+> <PUBCHEM_COMPOUND_CID>\n+2244\n+\n+> <PUBCHEM_CONFORMER_RMSD>\n+0.6\n+\n+> <PUBCHEM_CONFORMER_DIVERSEORDER>\n+1\n+11\n+10\n+3\n+15\n+17\n+13\n+5\n+16\n+7\n+14\n+9\n+8\n+4\n+18\n+6\n+12\n+2\n+\n+> <PUBCHEM_MMFF94_PARTIAL_CHARGES>\n+18\n+1 -0.23\n+10 -0.15\n+11 0.63\n+12 0.66\n+13 0.06\n+14 0.15\n+15 0.15\n+16 0.15\n+17 0.15\n+2 -0.65\n+21 0.5\n+3 -0.57\n+4 -0.57\n+5 0.08\n+6 0.09\n+7 -0.15\n+8 -0.15\n+9 -0.15\n+\n+> <PUBCHEM_EFFECTIVE_ROTOR_COUNT>\n+3\n+\n+> <PUBCHEM_CONFORMER_ID>\n+000008C400000001\n+\n+> <PUBCHEM_MMFF94_ENERGY>\n+39.5952\n+\n+> <PUBCHEM_FEATURE_SELFOVERLAP>\n+25.432\n+\n+> <PUBCHEM_SHAPE_FINGERPRINT>\n+1 1 18265615372930943622\n+100427 49 16967750034970055351\n+12138202 97 18271247217817981012\n+12423570 1 16692715976000295083\n+12524768 44 16753525617747228747\n+12716758 59 18341332292274886536\n+13024252 1 17968377969333732145\n+14181834 199 17830728755827362645\n+14614273 12 18262232214645093005\n+15207287 21 17703787037639964108\n+15775835 57 18340488876329928641\n+16945 1 18271533103414939405\n+193761 8 17907860604865584321\n+20645476 183 17677348215414174190\n+20871998 184 18198632231250704846\n+21040471 1 18411412921197846465\n+21501502 16 18123463883164380929\n+23402539 116 18271795865171824860\n+23419403 2 13539898140662769886\n+23552423 10 18048876295495619569\n+23559900 14 18272369794190581304\n+241688 4 16179044415907240795\n+257057 1 17478316999871287486\n+2748010 2 18339085878070479087\n+305870 269 18263645056784260212\n+528862 383 18117272558388284091\n+53812653 8 18410289211719108569\n+7364860 26 17910392788380644719\n+81228 2 18050568744116491203\n+\n+> <PUBCHEM_SHAPE_MULTIPOLES>\n+244.06\n+3.86\n+2.45\n+0.89\n+1.95\n+1.58\n+0.15\n+-1.85\n+0.38\n+-0.61\n+-0.02\n+0.29\n+0.01\n+-0.33\n+\n+> <PUBCHEM_SHAPE_SELFOVERLAP>\n+513.037\n+\n+> <PUBCHEM_SHAPE_VOLUME>\n+136\n+\n+> <PUBCHEM_COORDINATE_TYPE>\n+2\n+5\n+10\n+\n+$$$$\n+2244\n+ OpenBabel09021316243D\n+\n+ 21 21 0 0 0 0 0 0 0 0999 V2000\n+ 1.2333 0.5540 0.7792 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.6952 -2.71'..b'0 0 0 0 0 0 0 0 0 0\n+ -2.7247 -1.3605 -0.4564 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.5797 2.8872 0.0506 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -3.8374 0.8238 -0.5090 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.7290 1.4184 0.8593 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 4.2045 0.6969 -0.6924 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.7105 -0.3659 0.6426 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.2555 -3.5916 -0.7337 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 1 5 1 0 0 0 0\n+ 1 12 1 0 0 0 0\n+ 2 11 1 0 0 0 0\n+ 2 21 1 0 0 0 0\n+ 3 11 2 0 0 0 0\n+ 4 12 2 0 0 0 0\n+ 5 6 1 0 0 0 0\n+ 5 7 2 0 0 0 0\n+ 6 8 2 0 0 0 0\n+ 6 11 1 0 0 0 0\n+ 7 9 1 0 0 0 0\n+ 7 14 1 0 0 0 0\n+ 8 10 1 0 0 0 0\n+ 8 15 1 0 0 0 0\n+ 9 10 2 0 0 0 0\n+ 9 16 1 0 0 0 0\n+ 10 17 1 0 0 0 0\n+ 12 13 1 0 0 0 0\n+ 13 18 1 0 0 0 0\n+ 13 19 1 0 0 0 0\n+ 13 20 1 0 0 0 0\n+M END\n+> <PUBCHEM_COMPOUND_CID>\n+2244\n+\n+> <PUBCHEM_CONFORMER_RMSD>\n+0.6\n+\n+> <PUBCHEM_CONFORMER_DIVERSEORDER>\n+1\n+11\n+10\n+3\n+15\n+17\n+13\n+5\n+16\n+7\n+14\n+9\n+8\n+4\n+18\n+6\n+12\n+2\n+\n+> <PUBCHEM_MMFF94_PARTIAL_CHARGES>\n+18\n+1 -0.23\n+10 -0.15\n+11 0.63\n+12 0.66\n+13 0.06\n+14 0.15\n+15 0.15\n+16 0.15\n+17 0.15\n+2 -0.65\n+21 0.5\n+3 -0.57\n+4 -0.57\n+5 0.08\n+6 0.09\n+7 -0.15\n+8 -0.15\n+9 -0.15\n+\n+> <PUBCHEM_EFFECTIVE_ROTOR_COUNT>\n+3\n+\n+> <PUBCHEM_PHARMACOPHORE_FEATURES>\n+5\n+1 2 acceptor\n+1 3 acceptor\n+1 4 acceptor\n+3 2 3 11 anion\n+6 5 6 7 8 9 10 rings\n+\n+> <PUBCHEM_HEAVY_ATOM_COUNT>\n+13\n+\n+> <PUBCHEM_ATOM_DEF_STEREO_COUNT>\n+0\n+\n+> <PUBCHEM_ATOM_UDEF_STEREO_COUNT>\n+0\n+\n+> <PUBCHEM_BOND_DEF_STEREO_COUNT>\n+0\n+\n+> <PUBCHEM_BOND_UDEF_STEREO_COUNT>\n+0\n+\n+> <PUBCHEM_ISOTOPIC_ATOM_COUNT>\n+0\n+\n+> <PUBCHEM_COMPONENT_COUNT>\n+1\n+\n+> <PUBCHEM_CACTVS_TAUTO_COUNT>\n+1\n+\n+> <PUBCHEM_CONFORMER_ID>\n+000008C400000001\n+\n+> <PUBCHEM_COORDINATE_TYPE>\n+2\n+5\n+10\n+\n+$$$$\n+\n+\n+ 21 21 0 0 0 0 0 0 0 0999 V2000\n+ 1.2333 0.5540 0.7792 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.6952 -2.7148 -0.7502 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ 0.7958 -2.1843 0.8685 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ 1.7813 0.8105 -1.4821 O 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.0857 0.6088 0.4403 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.7927 -0.5515 0.1244 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.7288 1.8464 0.4133 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.1426 -0.4741 -0.2184 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.0787 1.9238 0.0706 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.7855 0.7636 -0.2453 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.1409 -1.8536 0.1477 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ 2.1094 0.6715 -0.3113 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.5305 0.5996 0.1635 C 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.1851 2.7545 0.6593 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.7247 -1.3605 -0.4564 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -2.5797 2.8872 0.0506 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -3.8374 0.8238 -0.5090 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.7290 1.4184 0.8593 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 4.2045 0.6969 -0.6924 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 3.7105 -0.3659 0.6426 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ -0.2555 -3.5916 -0.7337 H 0 0 0 0 0 0 0 0 0 0 0 0\n+ 1 5 1 0 0 0 0\n+ 1 12 1 0 0 0 0\n+ 2 11 1 0 0 0 0\n+ 2 21 1 0 0 0 0\n+ 3 11 2 0 0 0 0\n+ 4 12 2 0 0 0 0\n+ 5 6 1 0 0 0 0\n+ 5 7 2 0 0 0 0\n+ 6 8 2 0 0 0 0\n+ 6 11 1 0 0 0 0\n+ 7 9 1 0 0 0 0\n+ 7 14 1 0 0 0 0\n+ 8 10 1 0 0 0 0\n+ 8 15 1 0 0 0 0\n+ 9 10 2 0 0 0 0\n+ 9 16 1 0 0 0 0\n+ 10 17 1 0 0 0 0\n+ 12 13 1 0 0 0 0\n+ 13 18 1 0 0 0 0\n+ 13 19 1 0 0 0 0\n+ 13 20 1 0 0 0 0\n+M END\n+$$$$\n'

diff -r 750c1684d47c -r d150ac3d853d test-data/mol_0.sdf
--- /dev/null Thu Jan 01 00:00:00 1970 +0000
+++ b/test-data/mol_0.sdf Wed Aug 28 10:55:25 2019 -0400