Next changeset 1:6baac361495b (2020-10-31) |
Commit message:
"planemo upload for repository https://github.com/galaxyecology/tools-ecology/tree/master/tools/data_manipulation/xarray/ commit f1455c158011dc4aab0fd469cf794be6f4142992" |
added:
README.md test-data/Metadata_infos_from_dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc.Variables.tab test-data/Test1.tabular test-data/Test2.tabular test-data/dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc test-data/info_file.txt test-data/var_tab_dataset-ibi xarray_select.xml xarray_tool.py |
b |
diff -r 000000000000 -r 225d0d275a24 README.md --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.md Fri May 22 05:19:15 2020 -0400 |
b |
@@ -0,0 +1,8 @@ +# Xarray tools for netCDF +## netCDF metadata information + +The first tool `xarray_metadata_info ` uses xarray to provide users with general information about variable names, dimensions +and attributes. +Variables that can be extracted and dimensions available are printed in a tabular file. + +The tool also print a general information file. It's the result of the xarray method info(). |
b |
diff -r 000000000000 -r 225d0d275a24 test-data/Metadata_infos_from_dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc.Variables.tab --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/Metadata_infos_from_dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc.Variables.tab Fri May 22 05:19:15 2020 -0400 |
b |
@@ -0,0 +1,8 @@ +VariableName NumberOfDimensions Dim0Name Dim0Size Dim1Name Dim1Size Dim2Name Dim2Size Dim3Name Dim3Size +phy 4 time 145 depth 1 latitude 97 longitude 103 +chl 4 time 145 depth 1 latitude 97 longitude 103 +nh4 4 time 145 depth 1 latitude 97 longitude 103 +time 1 time 145 +longitude 1 longitude 103 +latitude 1 latitude 97 +depth 1 depth 1 |
b |
diff -r 000000000000 -r 225d0d275a24 test-data/Test1.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/Test1.tabular Fri May 22 05:19:15 2020 -0400 |
b |
b'@@ -0,0 +1,146 @@\n+\ttime\tdepth\tlongitude\tlatitude\tphy\n+0\t2002-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.0500183\n+1\t2003-01-15\t0.5057600140571594\t-2.0000007\t44.0\t1.25\n+2\t2003-02-15\t0.5057600140571594\t-2.0000007\t44.0\t1.3000183\n+3\t2003-03-15\t0.5057600140571594\t-2.0000007\t44.0\t6.0599976\n+4\t2003-04-15\t0.5057600140571594\t-2.0000007\t44.0\t2.25\n+5\t2003-05-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6499939\n+6\t2003-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.42999268\n+7\t2003-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.42999268\n+8\t2003-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+9\t2003-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+10\t2003-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+11\t2003-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.9299927\n+12\t2003-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.3900146\n+13\t2004-01-15\t0.5057600140571594\t-2.0000007\t44.0\t1.7400208\n+14\t2004-02-15\t0.5057600140571594\t-2.0000007\t44.0\t4.5\n+15\t2004-03-15\t0.5057600140571594\t-2.0000007\t44.0\t5.5500183\n+16\t2004-04-15\t0.5057600140571594\t-2.0000007\t44.0\t5.3099976\n+17\t2004-05-15\t0.5057600140571594\t-2.0000007\t44.0\t3.75\n+18\t2004-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.77001953\n+19\t2004-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+20\t2004-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+21\t2004-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.4500122\n+22\t2004-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+23\t2004-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.83999634\n+24\t2004-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.7400208\n+25\t2005-01-15\t0.5057600140571594\t-2.0000007\t44.0\t1.7700195\n+26\t2005-02-15\t0.5057600140571594\t-2.0000007\t44.0\t1.5500183\n+27\t2005-03-15\t0.5057600140571594\t-2.0000007\t44.0\t7.149994\n+28\t2005-04-15\t0.5057600140571594\t-2.0000007\t44.0\t3.649994\n+29\t2005-05-15\t0.5057600140571594\t-2.0000007\t44.0\t2.5200195\n+30\t2005-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+31\t2005-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6700134\n+32\t2005-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.4500122\n+33\t2005-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+34\t2005-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+35\t2005-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6199951\n+36\t2005-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.1499939\n+37\t2006-01-15\t0.5057600140571594\t-2.0000007\t44.0\t3.5299988\n+38\t2006-02-15\t0.5057600140571594\t-2.0000007\t44.0\t7.1799927\n+39\t2006-03-15\t0.5057600140571594\t-2.0000007\t44.0\t6.5599976\n+40\t2006-04-15\t0.5057600140571594\t-2.0000007\t44.0\t3.8000183\n+41\t2006-05-15\t0.5057600140571594\t-2.0000007\t44.0\t0.95999146\n+42\t2006-06-15\t0.5057600140571594\t-2.0000007\t44.0\t1.5\n+43\t2006-07-15\t0.5057600140571594\t-2.0000007\t44.0\t1.0299988\n+44\t2006-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+45\t2006-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.49002075\n+46\t2006-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+47\t2006-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.9299927\n+48\t2006-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.2099915\n+49\t2007-01-15\t0.5057600140571594\t-2.0000007\t44.0\t1.1499939\n+50\t2007-02-15\t0.5057600140571594\t-2.0000007\t44.0\t1.7000122\n+51\t2007-03-15\t0.5057600140571594\t-2.0000007\t44.0\t5.230011\n+52\t2007-04-15\t0.5057600140571594\t-2.0000007\t44.0\t3.8600159\n+53\t2007-05-15\t0.5057600140571594\t-2.0000007\t44.0\t0.83999634\n+54\t2007-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6799927\n+55\t2007-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+56\t2007-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+57\t2007-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+58\t2007-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.89001465\n+59\t2007-11-15\t0.5057600140571594\t-2.0000007\t44.0\t2.0299988\n+60\t2007-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.8399963\n+61\t2008-01-15\t0.5057600140571594\t-2.0000007\t44.0\t1.3399963\n+62\t2008-02-15\t0.5057600140571594\t-2.0000007\t44.0\t3.149994\n+63\t2008-03-15\t0.5057600140571594\t-2.0000007\t44.0\t4.5899963\n+64\t2008-04-15\t0.5057600140571594\t-2.0000007\t44.0\t5.080017\n+65\t2008-05-15\t0.5057600140571594\t-2.0000007\t44.0\t1.0\n+66\t2008-06-15\t0.5057600140571594\t-2.000'..b'2.0000007\t44.0\t1.4900208\n+79\t2009-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+80\t2009-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+81\t2009-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5800171\n+82\t2009-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6499939\n+83\t2009-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.8999939\n+84\t2009-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.3099976\n+85\t2010-01-15\t0.5057600140571594\t-2.0000007\t44.0\t1.5299988\n+86\t2010-02-15\t0.5057600140571594\t-2.0000007\t44.0\t2.9599915\n+87\t2010-03-15\t0.5057600140571594\t-2.0000007\t44.0\t5.450012\n+88\t2010-04-15\t0.5057600140571594\t-2.0000007\t44.0\t7.5899963\n+89\t2010-05-15\t0.5057600140571594\t-2.0000007\t44.0\t1.8000183\n+90\t2010-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+91\t2010-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+92\t2010-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+93\t2010-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.49002075\n+94\t2010-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+95\t2010-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.9299927\n+96\t2010-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.1499939\n+97\t2011-01-15\t0.5057600140571594\t-2.0000007\t44.0\t2.4900208\n+98\t2011-02-15\t0.5057600140571594\t-2.0000007\t44.0\t5.1799927\n+99\t2011-03-15\t0.5057600140571594\t-2.0000007\t44.0\t7.029999\n+100\t2011-04-15\t0.5057600140571594\t-2.0000007\t44.0\t2.4900208\n+101\t2011-05-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6499939\n+102\t2011-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.52001953\n+103\t2011-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+104\t2011-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.75\n+105\t2011-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+106\t2011-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+107\t2011-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.730011\n+108\t2011-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.0299988\n+109\t2012-01-15\t0.5057600140571594\t-2.0000007\t44.0\t3.149994\n+110\t2012-02-15\t0.5057600140571594\t-2.0000007\t44.0\t2.3099976\n+111\t2012-03-15\t0.5057600140571594\t-2.0000007\t44.0\t5.5200195\n+112\t2012-04-15\t0.5057600140571594\t-2.0000007\t44.0\t3.399994\n+113\t2012-05-15\t0.5057600140571594\t-2.0000007\t44.0\t3.7000122\n+114\t2012-06-15\t0.5057600140571594\t-2.0000007\t44.0\t2.5899963\n+115\t2012-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+116\t2012-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.4500122\n+117\t2012-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+118\t2012-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.61001587\n+119\t2012-11-15\t0.5057600140571594\t-2.0000007\t44.0\t2.0299988\n+120\t2012-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.4200134\n+121\t2013-01-15\t0.5057600140571594\t-2.0000007\t44.0\t2.2700195\n+122\t2013-02-15\t0.5057600140571594\t-2.0000007\t44.0\t7.0\n+123\t2013-03-15\t0.5057600140571594\t-2.0000007\t44.0\t10.550018\n+124\t2013-04-15\t0.5057600140571594\t-2.0000007\t44.0\t5.8399963\n+125\t2013-05-15\t0.5057600140571594\t-2.0000007\t44.0\t1.2400208\n+126\t2013-06-15\t0.5057600140571594\t-2.0000007\t44.0\t4.1700134\n+127\t2013-07-15\t0.5057600140571594\t-2.0000007\t44.0\t3.2099915\n+128\t2013-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+129\t2013-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+130\t2013-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.49002075\n+131\t2013-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.7799988\n+132\t2013-12-15\t0.5057600140571594\t-2.0000007\t44.0\t1.4500122\n+133\t2014-01-15\t0.5057600140571594\t-2.0000007\t44.0\t0.95999146\n+134\t2014-02-15\t0.5057600140571594\t-2.0000007\t44.0\t1.3900146\n+135\t2014-03-15\t0.5057600140571594\t-2.0000007\t44.0\t5.779999\n+136\t2014-04-15\t0.5057600140571594\t-2.0000007\t44.0\t5.4299927\n+137\t2014-05-15\t0.5057600140571594\t-2.0000007\t44.0\t1.1799927\n+138\t2014-06-15\t0.5057600140571594\t-2.0000007\t44.0\t0.730011\n+139\t2014-07-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+140\t2014-08-15\t0.5057600140571594\t-2.0000007\t44.0\t0.45999146\n+141\t2014-09-15\t0.5057600140571594\t-2.0000007\t44.0\t0.5\n+142\t2014-10-15\t0.5057600140571594\t-2.0000007\t44.0\t0.6199951\n+143\t2014-11-15\t0.5057600140571594\t-2.0000007\t44.0\t0.480011\n+144\t2014-12-15\t0.5057600140571594\t-2.0000007\t44.0\t0.55999756\n' |
b |
diff -r 000000000000 -r 225d0d275a24 test-data/Test2.tabular --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/Test2.tabular Fri May 22 05:19:15 2020 -0400 |
b |
@@ -0,0 +1,25 @@ + time depth latitude longitude nh4 +0 2003-12-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 81.27 +1 2003-12-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 78.08 +2 2003-12-15 0.5057600140571594 45.5 -0.9166674017906189 55.149998 +3 2004-01-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 65.2 +4 2004-01-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 64.11 +5 2004-02-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 51.0 +6 2004-02-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 51.32 +7 2004-05-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 54.53 +8 2004-06-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 79.79 +9 2004-06-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 61.52 +10 2004-07-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 99.159996 +11 2004-07-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 77.93 +12 2004-08-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 110.149994 +13 2004-08-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 86.759995 +14 2004-09-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 112.369995 +15 2004-09-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 91.979996 +16 2004-10-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 109.63 +17 2004-10-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 95.509995 +18 2004-11-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 98.45 +19 2004-11-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 93.11 +20 2004-11-15 0.5057600140571594 45.5 -0.9166674017906189 56.78 +21 2004-12-15 0.5057600140571594 45.166664123535156 -0.6666674017906189 84.25 +22 2004-12-15 0.5057600140571594 45.416664123535156 -0.8333340883255005 81.83 +23 2004-12-15 0.5057600140571594 45.5 -0.9166674017906189 57.07 |
b |
diff -r 000000000000 -r 225d0d275a24 test-data/dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc |
b |
Binary file test-data/dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc has changed |
b |
diff -r 000000000000 -r 225d0d275a24 test-data/info_file.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/info_file.txt Fri May 22 05:19:15 2020 -0400 |
[ |
@@ -0,0 +1,74 @@ +xarray.Dataset { +dimensions: + depth = 1 ; + latitude = 97 ; + longitude = 103 ; + time = 145 ; + +variables: + float32 phy(time, depth, latitude, longitude) ; + phy:_CoordinateAxes = time depth latitude longitude ; + phy:long_name = Mole Concentration of Phytoplankton expressed as carbon in sea water ; + phy:standard_name = mole_concentration_of_phytoplankton_expressed_as_carbon_in_sea_water ; + phy:units = mmol.m-3 ; + phy:unit_long = mole_concentration_of_phytoplankton_expressed_as_carbon_in_sea_water ; + datetime64[ns] time(time) ; + time:standard_name = time ; + time:long_name = time ; + time:_CoordinateAxisType = Time ; + time:axis = T ; + float32 chl(time, depth, latitude, longitude) ; + chl:_CoordinateAxes = time depth latitude longitude ; + chl:long_name = Mass Concentration of Chlorophyll in Sea Water ; + chl:standard_name = mass_concentration_of_chlorophyll_in_sea_water ; + chl:units = mg.m-3 ; + chl:unit_long = milligram of chlorophyll per cubic meter ; + float32 nh4(time, depth, latitude, longitude) ; + nh4:_CoordinateAxes = time depth latitude longitude ; + nh4:long_name = Mole Concentration of Ammonium in Sea Water ; + nh4:standard_name = mole_concentration_of_ammonium_in_sea_water ; + nh4:units = mmol.m-3 ; + nh4:unit_long = millimoles of Ammonium per cubic meter ; + float32 longitude(longitude) ; + longitude:long_name = Longitude ; + longitude:units = degrees_east ; + longitude:standard_name = longitude ; + longitude:axis = X ; + longitude:unit_long = Degrees East ; + longitude:step = 0.08333f ; + longitude:_CoordinateAxisType = Lon ; + float32 latitude(latitude) ; + latitude:long_name = Latitude ; + latitude:units = degrees_north ; + latitude:standard_name = latitude ; + latitude:axis = Y ; + latitude:unit_long = Degrees North ; + latitude:step = 0.08333f ; + latitude:_CoordinateAxisType = Lat ; + float32 depth(depth) ; + depth:long_name = Depth ; + depth:units = m ; + depth:axis = Z ; + depth:positive = down ; + depth:unit_long = Meters ; + depth:standard_name = depth ; + depth:_CoordinateAxisType = Height ; + depth:_CoordinateZisPositive = down ; + +// global attributes: + :title = CMEMS IBI REANALYSIS: MONTHLY BIOGEOCHEMICAL PRODUCTS (REGULAR GRID) ; + :institution = Puertos del Estado (PdE) - Mercator-Ocean (MO) ; + :references = http://marine.copernicus.eu ; + :source = CMEMS IBI-MFC ; + :Conventions = CF-1.0 ; + :history = Data extracted from dataset http://puertos2.cesga.es:8080/thredds/dodsC/dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid ; + :time_min = 7272.0 ; + :time_max = 112464.0 ; + :julian_day_unit = Hours since 2002-02-15 ; + :z_min = 0.5057600140571594 ; + :z_max = 0.5057600140571594 ; + :latitude_min = 43.0 ; + :latitude_max = 51.0 ; + :longitude_min = -6.000000476837158 ; + :longitude_max = 2.4999990463256836 ; +} \ No newline at end of file |
b |
diff -r 000000000000 -r 225d0d275a24 test-data/var_tab_dataset-ibi --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/var_tab_dataset-ibi Fri May 22 05:19:15 2020 -0400 |
b |
@@ -0,0 +1,7 @@ +time 1 time 145 +chl 4 time 145 depth 1 latitude 97 longitude 103 +nh4 4 time 145 depth 1 latitude 97 longitude 103 +longitude 1 longitude 103 +latitude 1 latitude 97 +depth 1 depth 1 +phy 4 time 145 depth 1 latitude 97 longitude 103 |
b |
diff -r 000000000000 -r 225d0d275a24 xarray_select.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xarray_select.xml Fri May 22 05:19:15 2020 -0400 |
[ |
b'@@ -0,0 +1,298 @@\n+<tool id="xarray_select" name="NetCDF xarray Selection" version="0.15.1">\n+ <description>extracts variable values with custom conditions on dimensions</description>\n+ <requirements>\n+ <requirement type="package" version="3">python</requirement>\n+ <requirement type="package" version="1.5.3">netcdf4</requirement>\n+ <requirement type="package" version="0.15.1">xarray</requirement>\n+ <requirement type="package" version="0.7.0">geopandas</requirement>\n+ <requirement type="package" version="1.7.0">shapely</requirement>\n+ </requirements>\n+ <command detect_errors="exit_code"><![CDATA[\n+ mkdir output_dir &&\n+ python \'$__tool_directory__/xarray_tool.py\' \'$input\' --select \'$var\'\n+ --verbose\n+ --filter\n+ #for $i,$uc in enumerate($user_choice)\n+ #if $uc.condi_between.comparator=="bi"\n+ \'${uc.dim}#${uc.condi_between.comparator}#${uc.condi_between.t1}#${uc.condi_between.t2}\'\n+ #else\n+ \'${uc.dim}#${uc.condi_between.comparator}#${uc.condi_between.value}\'\n+ #end if\n+ #end for\n+\n+ #if $time.condi_datetime.datetime=="yes"\n+ --time\n+ #if $time.condi_datetime.condi_between.comparator=="sl"\n+ \'${time.condi_datetime.dim}#${time.condi_datetime.condi_between.comparator}#${time.condi_datetime.condi_between.t1}#${time.condi_datetime.condi_between.t2}\'\n+ #else\n+ \'${time.condi_datetime.dim}#${time.condi_datetime.condi_between.comparator}#${time.condi_datetime.condi_between.t1}\'\n+ #end if\n+ #end if\n+\n+ #if $condi_source_coord.coord_source=="coord_from_file"\n+ --coords \'$coord_tabular\'\n+ --latname \'$condi_source_coord.lat_dim\' --lonname \'$condi_source_coord.lon_dim\'\n+ --outputdir output_dir\n+ #else\n+ --outfile \'final.tabular\'\n+ #if $condi_source_coord.condi_coord.coord==\'single\'\n+ --latname $condi_source_coord.condi_coord.lat_dim\n+ --latvalN $condi_source_coord.condi_coord.lat_val\n+ --lonname $condi_source_coord.condi_coord.lon_dim\n+ --lonvalE $condi_source_coord.condi_coord.lon_val\n+ #elif $condi_source_coord.condi_coord.coord==\'subregion\'\n+ --latname $condi_source_coord.condi_coord.lat_dim\n+ --latvalN $condi_source_coord.condi_coord.lat_valN\n+ --latvalS $condi_source_coord.condi_coord.lat_valS\n+ --lonname $condi_source_coord.condi_coord.lon_dim\n+ --lonvalE $condi_source_coord.condi_coord.lon_valE\n+ --lonvalW $condi_source_coord.condi_coord.lon_valW\n+ #end if\n+ #end if\n+ ]]></command>\n+ <inputs>\n+ <param type="data" name="input" label="Input netcdf file" format="netcdf"/>\n+ <param type="data" label="Tabular of variables" name="var_tab" format="tabular" help="Select the tabular file which summarize the available variables and dimensions."/>\n+\n+ <param name="var" type="select" label="Choose the variable to extract">\n+ <options from_dataset="var_tab">\n+ <column name="name" index="0"/>\n+ <column name="value" index="0"/>\n+ </options>\n+ </param>\n+\n+ <conditional name="condi_source_coord">\n+ <param name="coord_source" type="select" label="Source of coordinates">\n+ <option value="coord_from_stdin">Manually enter coordinates</option>\n+ <option value="coord_from_file">Use coordinates from input file</option>\n+ </param>\n+\n+ <when value="coord_from_file">\n+ <param type="data" label="Tabular of coord" name="coord_tabular" format="tabular" help="Format : Latitude\tLongitude"/>\n+ <param name="lat_dim" type="select" label="Name of latitude coordinate" >\n+ <options from_dataset="var_tab">\n+ <column name="value" index="0"/>\n+ </options>\n+ </param>\n+ '..b'impleoutput" from_work_dir="final.tabular" format="tabular">\n+ <filter>condi_source_coord[\'coord_source\'] == \'coord_from_stdin\'</filter>\n+ </data>\n+ </outputs>\n+ <tests>\n+ <test>\n+ <param name="input" value="dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc"/>\n+ <param name="var" value="phy"/>\n+ <param name="var_tab" value="var_tab_dataset-ibi"/>\n+ <conditional name="condi_source_coord">\n+ <param name="coord_source" value="coord_from_stdin"/>\n+ <conditional name="condi_coord">\n+ <param name="coord" value="single"/>\n+ <param name="lat_dim" value="latitude"/>\n+ <param name="lat_val" value="44.0"/>\n+ <param name="lon_dim" value="longitude"/>\n+ <param name="lon_val" value="-2.0"/>\n+ </conditional>\n+ </conditional>\n+ <output name="simpleoutput" value="Test1.tabular">\n+ <assert_contents>\n+ <has_text_matching expression="0\\t2002-12-15\\t0.5"/>\n+ <has_text_matching expression="144\\t2014-12-15\\t0.5"/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ <test>\n+ <param name="input" value="dataset-ibi-reanalysis-bio-005-003-monthly-regulargrid_1510914389133.nc"/>\n+ <param name="var_tab" value="var_tab_dataset-ibi"/>\n+ <param name="var" value="nh4"/>\n+\n+ <conditional name="condi_source_coord">\n+ <param name="coord_source" value="coord_from_stdin"/>\n+ <conditional name="condi_coord">\n+ <param name="coord" value="global"/>\n+ </conditional>\n+ </conditional>\n+ <section name="time">\n+ <conditional name="condi_datetime">\n+ <param name="datetime" value="yes"/>\n+ <conditional name="condi_between">\n+ <param name="comparator" value="sl"/>\n+ <param name="t1" value="2003-12-15" />\n+ <param name="t2" value="2004-12-15" />\n+ </conditional>\n+ </conditional>\n+ </section>\n+ <repeat name="user_choice">\n+ <param name="dim" value="nh4"/>\n+ <conditional name="condi_between">\n+ <param name="comparator" value="ge"/>\n+ <param name="value" value="50."/>\n+ </conditional>\n+ </repeat>\n+ <output name="simpleoutput" value="Test2.tabular">\n+ <assert_contents>\n+ <has_text_matching expression="0\\t2003-12-15\\t0.5"/>\n+ <has_text_matching expression="23\\t2004-12-15\\t0.5"/>\n+ </assert_contents>\n+ </output>\n+ </test>\n+ </tests>\n+\n+ <help><![CDATA[\n+**What it does**\n+\n+This tool extracts variable values with custom conditions on dimensions.\n+\n+It can use manualy given coordinates or automaticaly take them from a tabular file to filter informations.\n+\n+If no values are availables at a coordinate X, the tool will search the closest coordinate with a non NA value.\n+\n+Filter can be set on every dimension. Available filtering operations are : =, >, <, >=, <=, [interval], ]interval[.\n+\n+\n+\n+**Input**\n+\n+A netcdf file (.nc).\n+\n+Variable tabular file from \'Netcdf Metadate Info\'.\n+\n+Tabular file with coordinates and the following structure : \'lat\'\t\'lon\'.\n+\n+\n+**Outputs**\n+\n+A single output with values for the wanted variable if there is only one coordinate.\n+\n+A data collection where one file is created for every coordinate, if multiple coordinates from tabular file.\n+\n+\n+-------------------------------------------------\n+\n+The xarray select tool can be used after the xarray Info.\n+ ]]></help>\n+</tool>\n' |
b |
diff -r 000000000000 -r 225d0d275a24 xarray_tool.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/xarray_tool.py Fri May 22 05:19:15 2020 -0400 |
[ |
b'@@ -0,0 +1,302 @@\n+# xarray tool for:\n+# - getting metadata information\n+# - select data and save results in csv file for further post-processing\n+\n+import argparse\n+import csv\n+import warnings\n+\n+import geopandas as gdp\n+\n+import pandas as pd\n+\n+from shapely.geometry import Point\n+from shapely.ops import nearest_points\n+\n+import xarray as xr\n+\n+\n+class XarrayTool ():\n+ def __init__(self, infile, outfile_info="", outfile_summary="",\n+ select="", outfile="", outputdir="", latname="",\n+ latvalN="", latvalS="", lonname="", lonvalE="",\n+ lonvalW="", filter_list="", coords="", time="",\n+ verbose=False\n+ ):\n+ self.infile = infile\n+ self.outfile_info = outfile_info\n+ self.outfile_summary = outfile_summary\n+ self.select = select\n+ self.outfile = outfile\n+ self.outputdir = outputdir\n+ self.latname = latname\n+ if latvalN != "" and latvalN is not None:\n+ self.latvalN = float(latvalN)\n+ else:\n+ self.latvalN = ""\n+ if latvalS != "" and latvalS is not None:\n+ self.latvalS = float(latvalS)\n+ else:\n+ self.latvalS = ""\n+ self.lonname = lonname\n+ if lonvalE != "" and lonvalE is not None:\n+ self.lonvalE = float(lonvalE)\n+ else:\n+ self.lonvalE = ""\n+ if lonvalW != "" and lonvalW is not None:\n+ self.lonvalW = float(lonvalW)\n+ else:\n+ self.lonvalW = ""\n+ self.filter = filter_list\n+ self.time = time\n+ self.coords = coords\n+ self.verbose = verbose\n+ # initialization\n+ self.dset = None\n+ self.gset = None\n+ if self.verbose:\n+ print("infile: ", self.infile)\n+ print("outfile_info: ", self.outfile_info)\n+ print("outfile_summary: ", self.outfile_summary)\n+ print("outfile: ", self.outfile)\n+ print("select: ", self.select)\n+ print("outfile: ", self.outfile)\n+ print("outputdir: ", self.outputdir)\n+ print("latname: ", self.latname)\n+ print("latvalN: ", self.latvalN)\n+ print("latvalS: ", self.latvalS)\n+ print("lonname: ", self.lonname)\n+ print("lonvalE: ", self.lonvalE)\n+ print("lonvalW: ", self.lonvalW)\n+ print("filter: ", self.filter)\n+ print("time: ", self.time)\n+ print("coords: ", self.coords)\n+\n+ def info(self):\n+ f = open(self.outfile_info, \'w\')\n+ ds = xr.open_dataset(self.infile)\n+ ds.info(f)\n+ f.close()\n+\n+ def summary(self):\n+ f = open(self.outfile_summary, \'w\')\n+ ds = xr.open_dataset(self.infile)\n+ writer = csv.writer(f, delimiter=\'\\t\')\n+ header = [\'VariableName\', \'NumberOfDimensions\']\n+ for idx, val in enumerate(ds.dims.items()):\n+ header.append(\'Dim\'+str(idx)+\'Name\')\n+ header.append(\'Dim\'+str(idx)+\'Size\')\n+ writer.writerow(header)\n+ for name, da in ds.data_vars.items():\n+ line = [name]\n+ line.append(len(ds[name].shape))\n+ for d, s in zip(da.shape, da.sizes):\n+ line.append(s)\n+ line.append(d)\n+ writer.writerow(line)\n+ for name, da in ds.coords.items():\n+ line = [name]\n+ line.append(len(ds[name].shape))\n+ for d, s in zip(da.shape, da.sizes):\n+ line.append(s)\n+ line.append(d)\n+ writer.writerow(line)\n+ f.close()\n+\n+ def rowfilter(self, single_filter):\n+ split_filter = single_filter.split(\'#\')\n+ filter_varname = split_filter[0]\n+ op = split_filter[1]\n+ ll = float(split_filter[2])\n+ if (op == \'bi\'):\n+ rl = float(split_filter[3])\n+ if filter_varname == self.select:\n+ # filter on values of the selected variable\n+ '..b' else:\n+ self.gset = self.dset\n+\n+ def nearest_location(self):\n+ # Build a geopandas dataframe with all first elements in each dimension\n+ # so we assume null values correspond to a mask that is the same for\n+ # all dimensions in the dataset.\n+ dsel_frame = self.dset\n+ for dim in self.dset.dims:\n+ if dim != self.latname and dim != self.lonname:\n+ dsel_frame = dsel_frame.isel({dim: 0})\n+ # transform to pandas dataframe\n+ dff = dsel_frame.to_dataframe().dropna().reset_index()\n+ # transform to geopandas to collocate\n+ gdf = gdp.GeoDataFrame(dff,\n+ geometry=gdp.points_from_xy(dff[self.lonname],\n+ dff[self.latname]))\n+ # Find nearest location where values are not null\n+ point = Point(self.lonvalE, self.latvalN)\n+ multipoint = gdf.geometry.unary_union\n+ queried_geom, nearest_geom = nearest_points(point, multipoint)\n+ self.nearest_latvalN = nearest_geom.y\n+ self.nearest_lonvalE = nearest_geom.x\n+\n+ def selection_from_coords(self):\n+ fcoords = pd.read_csv(self.coords, sep=\'\\t\')\n+ for row in fcoords.itertuples():\n+ self.latvalN = row[0]\n+ self.lonvalE = row[1]\n+ self.outfile = (self.outputdir + \'/\' + self.select + \'_\'\n+ + str(row.Index) + \'.tabular\')\n+ self.selection()\n+\n+\n+if __name__ == \'__main__\':\n+ warnings.filterwarnings("ignore")\n+ parser = argparse.ArgumentParser()\n+\n+ parser.add_argument(\n+ \'infile\',\n+ help=\'netCDF input filename\'\n+ )\n+ parser.add_argument(\n+ \'--info\',\n+ help=\'Output filename where metadata information is stored\'\n+ )\n+ parser.add_argument(\n+ \'--summary\',\n+ help=\'Output filename where data summary information is stored\'\n+ )\n+ parser.add_argument(\n+ \'--select\',\n+ help=\'Variable name to select\'\n+ )\n+ parser.add_argument(\n+ \'--latname\',\n+ help=\'Latitude name\'\n+ )\n+ parser.add_argument(\n+ \'--latvalN\',\n+ help=\'North latitude value\'\n+ )\n+ parser.add_argument(\n+ \'--latvalS\',\n+ help=\'South latitude value\'\n+ )\n+ parser.add_argument(\n+ \'--lonname\',\n+ help=\'Longitude name\'\n+ )\n+ parser.add_argument(\n+ \'--lonvalE\',\n+ help=\'East longitude value\'\n+ )\n+ parser.add_argument(\n+ \'--lonvalW\',\n+ help=\'West longitude value\'\n+ )\n+ parser.add_argument(\n+ \'--coords\',\n+ help=\'Input file containing Latitude and Longitude\'\n+ \'for geographical selection\'\n+ )\n+ parser.add_argument(\n+ \'--filter\',\n+ nargs="*",\n+ help=\'Filter list variable#operator#value_s#value_e\'\n+ )\n+ parser.add_argument(\n+ \'--time\',\n+ help=\'select timeseries variable#operator#value_s[#value_e]\'\n+ )\n+ parser.add_argument(\n+ \'--outfile\',\n+ help=\'csv outfile for storing results of the selection\'\n+ \'(valid only when --select)\'\n+ )\n+ parser.add_argument(\n+ \'--outputdir\',\n+ help=\'folder name for storing results with multiple selections\'\n+ \'(valid only when --select)\'\n+ )\n+ parser.add_argument(\n+ "-v", "--verbose",\n+ help="switch on verbose mode",\n+ action="store_true"\n+ )\n+ args = parser.parse_args()\n+\n+ p = XarrayTool(args.infile, args.info, args.summary, args.select,\n+ args.outfile, args.outputdir, args.latname,\n+ args.latvalN, args.latvalS, args.lonname,\n+ args.lonvalE, args.lonvalW, args.filter,\n+ args.coords, args.time, args.verbose)\n+ if args.info:\n+ p.info()\n+ if args.summary:\n+ p.summary()\n+ if args.coords:\n+ p.selection_from_coords()\n+ elif args.select:\n+ p.selection()\n' |