| Previous changeset 20:ab27c4bd14b9 (2017-07-14) Next changeset 22:bed5018e7ae3 (2017-07-17) |
|
Commit message:
Uploaded |
|
modified:
filter_tabular.xml query_db.py query_tabular.xml sqlite_to_tabular.xml test-data/._query_results.tsv test-data/filtered_people_results.tsv test-data/filtered_pets_results.tsv test-data/pet_normalized_query_results.tsv test-data/query_results.tsv test-data/testdb.sqlite |
|
added:
test-data/._add_to_db_results.tsv test-data/add_to_db_results.tsv |
|
removed:
test-data/._filtered_people_results.tsv test-data/._filtered_pets_results.tsv |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d filter_tabular.xml --- a/filter_tabular.xml Fri Jul 14 11:39:27 2017 -0400 +++ b/filter_tabular.xml Fri Jul 14 17:34:22 2017 -0400 |
| [ |
| @@ -1,4 +1,4 @@ -<tool id="filter_tabular" name="Filter Tabular" version="5.0.0"> +<tool id="filter_tabular" name="Filter Tabular" version="1.0.0"> <description></description> <macros> @@ -7,11 +7,8 @@ <requirements> </requirements> - <stdio> - <exit_code range="1:" /> - </stdio> - <command><![CDATA[ - python $__tool_directory__/filter_tabular.py + <command detect_errors="exit_code"><![CDATA[ + python '$__tool_directory__/filter_tabular.py' -i '$input' -j '$filter_json' -o '$output' @@ -36,41 +33,72 @@ <test> <param name="input" ftype="tabular" value="pets.tsv"/> <repeat name="linefilters"> - <param name="filter_type" value="comment"/> - <param name="comment_char" value="35"/> + <conditional name="filter"> + <param name="filter_type" value="regex"/> + <param name="regex_pattern" value="^\d+"/> + <param name="regex_action" value="include_find"/> + </conditional> </repeat> <repeat name="linefilters"> - <param name="filter_type" value="append_line_num"/> + <conditional name="filter"> + <param name="filter_type" value="append_line_num"/> + </conditional> </repeat> <repeat name="linefilters"> - <param name="filter_type" value="select_columns"/> - <param name="columns" value="7,2,3,4,1"/> + <conditional name="filter"> + <param name="filter_type" value="select_columns"/> + <param name="columns" value="7,2,3,4,1"/> + </conditional> </repeat> <repeat name="linefilters"> - <param name="filter_type" value="replace"/> - <param name="column" value="c4"/> - <param name="regex_pattern" value="(\d+)/(\d+)/(\d+)"/> - <param name="regex_replace" value="19\3-\2-\1"/> + <conditional name="filter"> + <param name="filter_type" value="replace"/> + <param name="column" value="c4"/> + <param name="regex_pattern" value="(\d+)/(\d+)/(\d+)"/> + <param name="regex_replace" value="19\3-\2-\1"/> + </conditional> </repeat> <output name="output" file="filtered_people_results.tsv"/> </test> <test> <param name="input" ftype="tabular" value="pets.tsv"/> <repeat name="linefilters"> - <param name="filter_type" value="comment"/> - <param name="comment_char" value="35"/> + <conditional name="filter"> + <param name="filter_type" value="comment"/> + <param name="comment_char" value="35"/> + </conditional> </repeat> <repeat name="linefilters"> - <param name="filter_type" value="append_line_num"/> + <conditional name="filter"> + <param name="filter_type" value="regex"/> + <param name="regex_pattern" value="^\d+"/> + <param name="regex_action" value="include_find"/> + </conditional> + </repeat> + <repeat name="linefilters"> + <conditional name="filter"> + <param name="filter_type" value="append_line_num"/> + </conditional> </repeat> <repeat name="linefilters"> - <param name="filter_type" value="select_columns"/> - <param name="columns" value="c7,c5,c6"/> + <conditional name="filter"> + <param name="filter_type" value="select_columns"/> + <param name="columns" value="c7,c5,c6"/> + </conditional> </repeat> <repeat name="linefilters"> - <param name="filter_type" value="normalize"/> - <param name="columns" value="c2,c3"/> - <param name="separator" value=","/> + <conditional name="filter"> + <param name="filter_type" value="normalize"/> + <param name="columns" value="c2,c3"/> + <param name="separator" value=","/> + </conditional> + </repeat> + <repeat name="linefilters"> + <conditional name="filter"> + <param name="filter_type" value="regex"/> + <param name="regex_pattern" value="^\d+\t\t"/> + <param name="regex_action" value="exclude_match"/> + </conditional> </repeat> <output name="output" file="filtered_pets_results.tsv"/> </test> |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d query_db.py --- a/query_db.py Fri Jul 14 11:39:27 2017 -0400 +++ b/query_db.py Fri Jul 14 17:34:22 2017 -0400 |
| b |
| @@ -28,7 +28,7 @@ return re.sub(expr, replace, item) -def get_connection(sqlitedb_path, addfunctions=False): +def get_connection(sqlitedb_path, addfunctions=True): conn = sqlite.connect(sqlitedb_path) if addfunctions: conn.create_function("re_match", 2, regex_match) |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d query_tabular.xml --- a/query_tabular.xml Fri Jul 14 11:39:27 2017 -0400 +++ b/query_tabular.xml Fri Jul 14 17:34:22 2017 -0400 |
| [ |
| b'@@ -1,4 +1,4 @@\n-<tool id="query_tabular" name="Query Tabular" version="5.0.0">\n+<tool id="query_tabular" name="Query Tabular" version="1.0.0">\n <description>using sqlite sql</description>\n \n <macros>\n@@ -7,29 +7,27 @@\n \n <requirements>\n </requirements>\n- <stdio>\n- <exit_code range="1:" />\n- </stdio>\n- <command><![CDATA[\n- cat $query_file &&\n+\n+ <command detect_errors="exit_code"><![CDATA[\n+ cat \'$query_file\' &&\n #if $add_to_database.withdb: \n #if $save_db:\n- cp "$add_to_database.withdb" "$sqlitedb" &&\n+ cp \'$add_to_database.withdb\' \'$sqlitedb\' &&\n #else:\n- cp "$add_to_database.withdb" "$workdb" &&\n+ cp \'$add_to_database.withdb\' \'$workdb\' &&\n #end if \n #end if\n- python $__tool_directory__/query_tabular.py \n+ python \'$__tool_directory__/query_tabular.py\'\n #if $save_db\n- -s "$sqlitedb"\n+ -s \'$sqlitedb\'\n #else\n- -s $workdb\n+ -s \'$workdb\'\n #end if\n- -j $table_json\n+ -j \'$table_json\'\n #if $sqlquery:\n- -Q "$query_file" \n+ -Q \'$query_file\' \n $no_header\n- -o $output\n+ -o \'$output\'\n #end if\n ]]></command>\n <configfiles>\n@@ -145,17 +143,36 @@\n </data>\n </outputs>\n <tests>\n-\n <test>\n <repeat name="tables">\n <param name="table" ftype="tabular" value="customers.tsv"/>\n- <param name="table_name" value="customers"/>\n- <param name="col_names" value="CustomerID,FirstName,LastName,Email,DOB,Phone"/>\n+ <section name="input_opts">\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="comment"/>\n+ <param name="comment_char" value="35"/>\n+ </conditional>\n+ </repeat>\n+ </section>\n+ <section name="tbl_opts">\n+ <param name="table_name" value="customers"/>\n+ <param name="col_names" value="CustomerID,FirstName,LastName,Email,DOB,Phone"/>\n+ </section>\n </repeat>\n <repeat name="tables">\n <param name="table" ftype="tabular" value="sales.tsv"/>\n- <param name="table_name" value="sales"/>\n- <param name="col_names" value="CustomerID,Date,SaleAmount"/>\n+ <section name="input_opts">\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="comment"/>\n+ <param name="comment_char" value="35"/>\n+ </conditional>\n+ </repeat>\n+ </section>\n+ <section name="tbl_opts">\n+ <param name="table_name" value="sales"/>\n+ <param name="col_names" value="CustomerID,Date,SaleAmount"/>\n+ </section>\n </repeat>\n <param name="sqlquery" value="SELECT FirstName,LastName,sum(SaleAmount) as "TotalSales" FROM customers join sales on customers.CustomerID = sales.CustomerID GROUP BY customers.CustomerID ORDER BY TotalSales DESC"/>\n <output name="output" file="sales_results.tsv"/>\n@@ -164,10 +181,28 @@\n <test>\n <repeat name="tables">\n <param name="table" ftype="tabular" value="customers.tsv"/>\n- <param name="col_names" value=",FirstName,LastName,,DOB,"/>\n+ <section name="input_opts">\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="comment"/>\n+ <param'..b' </repeat>\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="select_columns"/>\n+ <param name="columns" value="7,2,3,4,1"/>\n+ </conditional>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="replace"/>\n+ <param name="column" value="c4"/>\n+ <param name="regex_pattern" value="(\\d+)/(\\d+)/(\\d+)"/>\n+ <param name="regex_replace" value="19\\3-\\2-\\1"/>\n+ </conditional>\n+ </repeat>\n+ </section>\n+ <section name="tbl_opts">\n+ <param name="table_name" value="people"/>\n+ <param name="col_names" value="id,first,last,dob,pets"/>\n+ </section>\n </repeat>\n <repeat name="tables">\n <param name="table" ftype="tabular" value="pets.tsv"/>\n- <repeat name="linefilters">\n- <param name="filter_type" value="comment"/>\n- <param name="comment_char" value="35"/>\n- </repeat>\n- <repeat name="linefilters">\n- <param name="filter_type" value="append_line_num"/>\n- </repeat>\n- <repeat name="linefilters">\n- <param name="filter_type" value="select_columns"/>\n- <param name="columns" value="c7,c5,c6"/>\n- </repeat>\n- <repeat name="linefilters">\n- <param name="filter_type" value="normalize"/>\n- <param name="columns" value="c2,c3"/>\n- <param name="separator" value=","/>\n- </repeat>\n- <param name="table_name" value="pet"/>\n- <param name="col_names" value="id,name,animal"/>\n+ <section name="input_opts">\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="regex"/>\n+ <param name="regex_pattern" value="^\\d+"/>\n+ <param name="regex_action" value="include_find"/>\n+ </conditional>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="append_line_num"/>\n+ </conditional>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="select_columns"/>\n+ <param name="columns" value="c7,c5,c6"/>\n+ </conditional>\n+ </repeat>\n+ <repeat name="linefilters">\n+ <conditional name="filter">\n+ <param name="filter_type" value="normalize"/>\n+ <param name="columns" value="c2,c3"/>\n+ <param name="separator" value=","/>\n+ </conditional>\n+ </repeat>\n+ </section>\n+ <section name="tbl_opts">\n+ <param name="table_name" value="pet"/>\n+ <param name="col_names" value="id,name,animal"/>\n+ </section>\n </repeat>\n <param name="sqlquery" value="SELECT people.id,first,last,dob,name,animal,pets FROM people JOIN pet ON people.id = pet.id WHERE animal = \'cat\'"/>\n <output name="output" file="pet_normalized_query_results.tsv"/>\n' |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d sqlite_to_tabular.xml --- a/sqlite_to_tabular.xml Fri Jul 14 11:39:27 2017 -0400 +++ b/sqlite_to_tabular.xml Fri Jul 14 17:34:22 2017 -0400 |
| [ |
| @@ -1,20 +1,17 @@ <?xml version="1.0"?> -<tool id="sqlite_to_tabular" name="SQLite to tabular" version="5.0.0"> +<tool id="sqlite_to_tabular" name="SQLite to tabular" version="1.0.0"> <description>for SQL query</description> <macros> <import>macros.xml</import> </macros> - <stdio> - <exit_code range="1:" level="fatal" description="Error" /> - </stdio> - <command><![CDATA[ - python $__tool_directory__/sqlite_to_tabular.py - --sqlitedb="$sqlitedb" - --query_file="$query_file" + <command detect_errors="exit_code"><![CDATA[ + python '$__tool_directory__/sqlite_to_tabular.py' + --sqlitedb='$sqlitedb' + --query_file='$query_file' $no_header - --output="$query_results" + --output='$query_results' ]]></command> <configfiles> <configfile name="query_file"> |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/._add_to_db_results.tsv |
| b |
| Binary file test-data/._add_to_db_results.tsv has changed |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/._filtered_people_results.tsv |
| b |
| Binary file test-data/._filtered_people_results.tsv has changed |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/._filtered_pets_results.tsv |
| b |
| Binary file test-data/._filtered_pets_results.tsv has changed |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/._query_results.tsv |
| b |
| Binary file test-data/._query_results.tsv has changed |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/add_to_db_results.tsv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/add_to_db_results.tsv Fri Jul 14 17:34:22 2017 -0400 |
| b |
| @@ -0,0 +1,3 @@ +#id first last pets quote +1 Paula Brown 2 Time flies like and arrow. Fruit flies like a banana. +2 Steven Jones 1 I would have wrtten less if I had more time |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/filtered_people_results.tsv --- a/test-data/filtered_people_results.tsv Fri Jul 14 11:39:27 2017 -0400 +++ b/test-data/filtered_people_results.tsv Fri Jul 14 17:34:22 2017 -0400 |
| b |
| @@ -1,5 +1,4 @@ -1 FirstName LastName DOB Pets -2 Paula Brown 1978-05-24 2 -3 Steven Jones 1974-04-04 1 -4 Jane Doe 1978-05-24 0 -5 James Smith 1980-10-20 1 +1 Paula Brown 1978-05-24 2 +2 Steven Jones 1974-04-04 1 +3 Jane Doe 1978-05-24 0 +4 James Smith 1980-10-20 1 |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/filtered_pets_results.tsv --- a/test-data/filtered_pets_results.tsv Fri Jul 14 11:39:27 2017 -0400 +++ b/test-data/filtered_pets_results.tsv Fri Jul 14 17:34:22 2017 -0400 |
| b |
| @@ -1,6 +1,4 @@ -1 PetNames PetType -2 Rex dog -2 Fluff cat -3 Allie cat -4 -5 Spot +1 Rex dog +1 Fluff cat +2 Allie cat +4 Spot |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/pet_normalized_query_results.tsv --- a/test-data/pet_normalized_query_results.tsv Fri Jul 14 11:39:27 2017 -0400 +++ b/test-data/pet_normalized_query_results.tsv Fri Jul 14 17:34:22 2017 -0400 |
| b |
| @@ -1,3 +1,3 @@ #id first last dob name animal pets -2 Paula Brown 1978-05-24 Fluff cat 2 -3 Steven Jones 1974-04-04 Allie cat 1 +1 Paula Brown 1978-05-24 Fluff cat 2 +2 Steven Jones 1974-04-04 Allie cat 1 |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/query_results.tsv --- a/test-data/query_results.tsv Fri Jul 14 11:39:27 2017 -0400 +++ b/test-data/query_results.tsv Fri Jul 14 17:34:22 2017 -0400 |
| b |
| @@ -5,13 +5,13 @@ ADAMTSL1 SLDMCISGL 1 9 30.5 0.698 26 SB PPAP2C FGMYCMVFL 2 10 2.3 0.501 222 WB PPAP2C FGMYCMVFL 2 10 6.55 0.501 222 WB -ADAMTSL1 MCISGLCQL 4 12 6.65 0.306 1823 None -ADAMTSL1 MCISGLCQL 4 12 14.0 0.306 1823 None -PPAP2C MYCMVFLVK 4 12 0.65 0.07 23399 None -PPAP2C MYCMVFLVK 4 12 54.0 0.07 23399 None -ADAMTSL1 LDMCISGLC 2 10 42.0 0.069 23677 None -ADAMTSL1 LDMCISGLC 2 10 76.0 0.069 23677 None -PPAP2C SFGMYCMVF 1 9 0.5 0.066 24390 None -PPAP2C SFGMYCMVF 1 9 45.0 0.066 24390 None -ADAMTSL1 DMCISGLCQ 3 11 64.5 0.042 31630 None -ADAMTSL1 DMCISGLCQ 3 11 97.0 0.042 31630 None +ADAMTSL1 MCISGLCQL 4 12 6.65 0.306 1823 +ADAMTSL1 MCISGLCQL 4 12 14.0 0.306 1823 +PPAP2C MYCMVFLVK 4 12 0.65 0.07 23399 +PPAP2C MYCMVFLVK 4 12 54.0 0.07 23399 +ADAMTSL1 LDMCISGLC 2 10 42.0 0.069 23677 +ADAMTSL1 LDMCISGLC 2 10 76.0 0.069 23677 +PPAP2C SFGMYCMVF 1 9 0.5 0.066 24390 +PPAP2C SFGMYCMVF 1 9 45.0 0.066 24390 +ADAMTSL1 DMCISGLCQ 3 11 64.5 0.042 31630 +ADAMTSL1 DMCISGLCQ 3 11 97.0 0.042 31630 |
| b |
| diff -r ab27c4bd14b9 -r 357fe86f245d test-data/testdb.sqlite |
| b |
| Binary file test-data/testdb.sqlite has changed |