| Previous changeset 17:c6e6e224509c (2017-06-22) Next changeset 19:9d9ab2c69014 (2017-07-05) |
|
Commit message:
Uploaded |
|
modified:
query_tabular.py query_tabular.xml |
|
added:
._query_tabular.py test-data/._pet_normalized_query_results.tsv test-data/pet_normalized_query_results.tsv |
| b |
| diff -r c6e6e224509c -r b9f797bf4f38 ._query_tabular.py |
| b |
| Binary file ._query_tabular.py has changed |
| b |
| diff -r c6e6e224509c -r b9f797bf4f38 query_tabular.py --- a/query_tabular.py Thu Jun 22 09:15:36 2017 -0400 +++ b/query_tabular.py Wed Jul 05 11:51:15 2017 -0400 |
| [ |
| @@ -91,6 +91,12 @@ self.func = lambda i,l: '%d\t%s' % (i,l) elif filter_dict['filter'] == 'append_line_num': self.func = lambda i,l: '%s\t%d' % (l.rstrip('\r\n'),i) + elif filter_dict['filter'] == 'prepend_text': + s = filter_dict['column_text'] + self.func = lambda i,l: '%s\t%s' % (s,l) + elif filter_dict['filter'] == 'append_text': + s = filter_dict['column_text'] + self.func = lambda i,l: '%s\t%s' % (l.rstrip('\r\n'),s) elif filter_dict['filter'] == 'skip': cnt = filter_dict['count'] self.func = lambda i,l: l if i > cnt else None |
| b |
| diff -r c6e6e224509c -r b9f797bf4f38 query_tabular.xml --- a/query_tabular.xml Thu Jun 22 09:15:36 2017 -0400 +++ b/query_tabular.xml Wed Jul 05 11:51:15 2017 -0400 |
| [ |
| b'@@ -1,4 +1,4 @@\n-<tool id="query_tabular" name="Query Tabular" version="3.1.0">\n+<tool id="query_tabular" name="Query Tabular" version="4.0.0">\n <description>using sqlite sql</description>\n \n <requirements>\n@@ -94,8 +94,8 @@\n #elif $fi.filter.filter_type == \'comment\':\n #set $filter_dict = dict()\n #set $filter_dict[\'filter\'] = \'regex\'\n- #set $filter_dict[\'pattern\'] = \'^\' + str($fi.filter.comment_char)\n- #set $filter_dict[\'action\'] = \'exclude\'\n+ #set $filter_dict[\'pattern\'] = \'^(%s).*$\' % \'|\'.join([chr(int(x)).replace(\'|\',\'[|]\') for x in (str($fi.filter.comment_char)).split(\',\')])\n+ #set $filter_dict[\'action\'] = \'exclude_match\'\n #silent $input_filters.append($filter_dict)\n #elif $fi.filter.filter_type == \'regex\':\n #set $filter_dict = dict()\n@@ -106,12 +106,12 @@\n #elif $fi.filter.filter_type == \'select_columns\':\n #set $filter_dict = dict()\n #set $filter_dict[\'filter\'] = str($fi.filter.filter_type)\n- #set $filter_dict[\'columns\'] = [int(str($ci)) for $ci in str($fi.filter.columns).split(\',\')]\n+ #set $filter_dict[\'columns\'] = [int(str($ci).replace(\'c\',\'\')) for $ci in str($fi.filter.columns).split(\',\')]\n #silent $input_filters.append($filter_dict)\n #elif $fi.filter.filter_type == \'replace\':\n #set $filter_dict = dict()\n #set $filter_dict[\'filter\'] = str($fi.filter.filter_type)\n- #set $filter_dict[\'column\'] = int(str($fi.filter.column))\n+ #set $filter_dict[\'column\'] = int(str($fi.filter.column).replace(\'c\',\'\'))\n #set $filter_dict[\'pattern\'] = str($fi.filter.regex_pattern)\n #set $filter_dict[\'replace\'] = str($fi.filter.regex_replace)\n #silent $input_filters.append($filter_dict)\n@@ -119,10 +119,15 @@\n #set $filter_dict = dict()\n #set $filter_dict[\'filter\'] = str($fi.filter.filter_type)\n #silent $input_filters.append($filter_dict)\n+ #elif str($fi.filter.filter_type).endswith(\'pend_text\'):\n+ #set $filter_dict = dict()\n+ #set $filter_dict[\'filter\'] = str($fi.filter.filter_type)\n+ #set $filter_dict[\'column_text\'] = str($fi.filter.column_text)\n+ #silent $input_filters.append($filter_dict)\n #elif $fi.filter.filter_type == \'normalize\':\n #set $filter_dict = dict()\n #set $filter_dict[\'filter\'] = str($fi.filter.filter_type)\n- #set $filter_dict[\'columns\'] = [int(str($ci)) for $ci in str($fi.filter.columns).split(\',\')]\n+ #set $filter_dict[\'columns\'] = [int(str($ci).replace(\'c\',\'\')) for $ci in str($fi.filter.columns).split(\',\')]\n #set $filter_dict[\'separator\'] = str($fi.filter.separator)\n #silent $input_filters.append($filter_dict)\n #end if\n@@ -154,6 +159,8 @@\n <option value="replace">regex replace value in column</option>\n <option value="prepend_line_num">prepend a line number column</option>\n <option value="append_line_num">append a line number column</option>\n+ <option value="prepend_text">prepend a column with the given text</option>\n+ <option value="append_text">append a column with the given text</option>\n <option value="normalize">normalize list columns, replicates row for each item in list</option>\n </param>\n <when value="skip">\n@@ -161,12 +168,36 @@\n help="Leave blank to use the comment lines metadata for this dataset" />\n </when>\n <when value="comment">\n- <param name="comment_char" type="text" value="#" label="Comment line starting text">\n- <sanitizer sanitize="False"/>\n+ <param name="comment_char" type="select" display="checkboxes" multiple="True" label="Ignore lines beginning with these characters" help="lines beginning with these are skipped">\n+ '..b' </param>\n@@ -195,8 +228,13 @@\n </param>\n </when>\n <when value="normalize">\n- <param name="columns" type="data_column" data_ref="table" multiple="True" label="Columns to split"\n- help="If multiple columns are selected, they should have the same length and separator on each line"/>\n+ <param name="columns" type="text" value="" label="enter column numbers to normalize">\n+ <help><![CDATA[\n+ example: 2,4 or c2,c4 (selects the second, and fourth columns)\n+ If multiple columns are selected, they should have the same length and separator on each line\n+ ]]></help>\n+ <validator type="regex" message="Column ordinal positions separated by commas">^(c?[1-9]\\d*)(,c?[1-9]\\d*)*$</validator>\n+ </param>\n <param name="separator" type="text" value="," label="List item delimiter in column">\n <sanitizer sanitize="False"/>\n <validator type="regex" message="Anything but TAB or Newline">^[^\\t\\n\\r\\f\\v]+$</validator>\n@@ -300,6 +338,55 @@\n <output name="output" file="query_results.tsv"/>\n </test>\n \n+ <test>\n+ <repeat name="tables">\n+ <param name="table" ftype="tabular" value="pets.tsv"/>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="comment"/>\n+ <param name="comment_char" value="35"/>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="append_line_num"/>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="select_columns"/>\n+ <param name="columns" value="7,2,3,4,1"/>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="replace"/>\n+ <param name="column" value="c4"/>\n+ <param name="regex_pattern" value="(\\d+)/(\\d+)/(\\d+)"/>\n+ <param name="regex_replace" value="19\\3-\\2-\\1"/>\n+ </repeat>\n+ <param name="table_name" value="people"/>\n+ <param name="col_names" value="id,first,last,dob,pets"/>\n+\n+ </repeat>\n+ <repeat name="tables">\n+ <param name="table" ftype="tabular" value="pets.tsv"/>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="comment"/>\n+ <param name="comment_char" value="35"/>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="append_line_num"/>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="select_columns"/>\n+ <param name="columns" value="c7,c5,c6"/>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <param name="filter_type" value="normalize"/>\n+ <param name="columns" value="c2,c3"/>\n+ <param name="separator" value=","/>\n+ </repeat>\n+ <param name="table_name" value="pet"/>\n+ <param name="col_names" value="id,name,animal"/>\n+ </repeat>\n+ <param name="sqlquery" value="SELECT people.id,first,last,dob,name,animal,pets FROM people JOIN pet ON people.id = pet.id WHERE animal = \'cat\'"/>\n+ <output name="output" file="pet_normalized_query_results.tsv"/>\n+ </test>\n+\n </tests>\n <help><![CDATA[\n =============\n' |
| b |
| diff -r c6e6e224509c -r b9f797bf4f38 test-data/._pet_normalized_query_results.tsv |
| b |
| Binary file test-data/._pet_normalized_query_results.tsv has changed |
| b |
| diff -r c6e6e224509c -r b9f797bf4f38 test-data/pet_normalized_query_results.tsv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/pet_normalized_query_results.tsv Wed Jul 05 11:51:15 2017 -0400 |
| b |
| @@ -0,0 +1,3 @@ +#id first last dob name animal pets +2 Paula Brown 1978-05-24 Fluff cat 2 +3 Steven Jones 1974-04-04 Allie cat 1 |