# HG changeset patch # User climate # Date 1640877849 0 # Node ID 81b0ca76435dce311034ff868739368f84d3ebd6 "planemo upload for repository https://gitlab.com/eetun-tiimi/EODIE commit c4a5672398bc878dd2bc0bf4f3a26f59b3f6395c" diff -r 000000000000 -r 81b0ca76435d eodie.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/eodie.xml Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,363 @@ + + converts simple features data between file formats + + macros.xml + + + + eodie + python + shapely + rasterio + rasterstats + fiona + numpy + gdal + pyyaml + unzip + tar + + + + + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + +
+ + + + + + + + + + + + + + + + + + + + +
+
+ + +
diff -r 000000000000 -r 81b0ca76435d macros.xml --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/macros.xml Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,223 @@ + + 1.0.2 + + + topic_0610 + topic_3050 + + + + + + + + + + + + + 10.5281/zenodo.4762323 + + + diff -r 000000000000 -r 81b0ca76435d postprocess_csv.sh --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/postprocess_csv.sh Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +WORKDIR=$1 +RESDIR=$2 + +echo "====================================================================" +echo " Convert EODIE csv to tabular " +echo "====================================================================" + +nb_csv=$(find $WORKDIR -type f -name "*.csv" | wc -l) +echo "Number of csv file to convert: $nb_csv" +if [[ $nb_csv -gt 0 ]]; then + echo "Start" + for infile in $WORKDIR/*.csv; do + echo "processing $infile" + sed -i.bak -e "s/,/\t/g" $infile + mv $infile $RESDIR/. + done +fi + +echo "EODIE Tabular saved." diff -r 000000000000 -r 81b0ca76435d test-data/20211213-104427.log --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/20211213-104427.log Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,6 @@ +INFO:root:All inputs for this process: dict_items([('platform', 'tif'), ('config', {'platform': 'tif', 'datepattern': '20[1-2][0-9][0-1][0-9][0-3][0-9]', 'name': 'testrgb', 'maxcloudcover': 99, 'pixelsize': 10, 'resampling_method': 'bilinear', 'fieldname': 'Name', 'lookup': './postprocesses/lookuptable.txt'}), ('mydir', None), ('myfile', 'smaller_area_20100401.tif'), ('input', ['smaller_area_20100401.tif']), ('shpbase', 'shapefile'), ('outpath', './results'), ('idname', 'id'), ('statistics_out', True), ('array_out', False), ('indexlist', None), ('statistics', ['count', 'mean', 'percentile_10']), ('startdate', '20160101'), ('enddate', '20211221'), ('keep_shp', False), ('geotiff_out', False), ('test', False), ('exclude_border', False), ('extmask', None), ('exclude_splitshp', True), ('verbose', False), ('format', ['statistics'])]) +INFO:root:File to be processed smaller_area_20100401.tif +INFO:root:Checking the projection of the inputfile now +INFO:root:Reprojectcommand: ogr2ogr -t_srs EPSG:3067 shapefile_reprojected_3067.shp shapefile.shp +INFO:root:input shapefile had other than EPSG 3067 but was reprojected and works now +INFO:root:stat to csv in: ./results/testrgb_20100401__statistics.csv diff -r 000000000000 -r 81b0ca76435d test-data/20211217-210343.log --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/20211217-210343.log Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,21 @@ +INFO:root:All inputs for this process: dict_items([('platform', 's2'), ('config', {'platform': 's2', 'bitmask': 0, 'tobemaskedlist': [9, 8, 3, 10, 0, 1], 'filepattern': 'S2[A-C]_MSIL2A_\\d{8}T\\d{6}_N\\d{4}_R\\d{3}_T\\d{2}[A-Z]{3}_\\d{8}T\\d{6}.SAFE$', 'red': 'B04', 'green': 'B03', 'blue': 'B02', 'nir': 'B08', 'r_edge': 'B05', 'swir1': 'B11', 'swir2': 'B12', 'productnameidentifier': 'S2*.SAFE', 'bandlocation': ['.', '*', '*', 'IMG_DATA'], 'pathbuildinglist': ['R', 'pixelsize', 'm', '*', 'bandname', '_', 'pixelsize', 'm.jp2'], 'cloudfilename': 'SCL', 'tilepattern': '(?<=T)[0-9]{2}[A-Z]{3}', 'datepattern': '20[1-2][0-9][0-1][0-9][0-3][0-9]', 'band_designation': 'B[0-1]?\\dA?', 'quantification_value': 10000, 'B01': [60], 'B02': [10, 20, 60], 'B03': [10, 20, 60], 'B04': [10, 20, 60], 'B05': [20, 60], 'B06': [20, 60], 'B07': [20, 60], 'B08': [10], 'B8A': [20, 60], 'B09': [60], 'B11': [20, 60], 'B12': [20, 60], 'SCL': [20, 60], 'AOT': [10, 20, 60], 'TCI': [10, 20, 60], 'WVP': [10, 20, 60], 'maxcloudcover': 99, 'pixelsize': 10, 'resampling_method': 'bilinear', 'tileshp': './sentinel2_tiles_world/sentinel2_tiles_world', 'fieldname': 'Name', 'lookup': './postprocesses/lookuptable.txt'}), ('mydir', 'data_collection'), ('myfile', None), ('input', ['data_collection/S2B_MSIL2A_20200626T095029_N0214_R079_T34VFN_20200626T123234.SAFE']), ('shpbase', 'shapefile'), ('outpath', './results'), ('idname', 'ID'), ('statistics_out', True), ('array_out', False), ('indexlist', ['ndvi']), ('statistics', ['count', 'mean', 'median', 'std']), ('startdate', '20160101'), ('enddate', '20211221'), ('keep_shp', False), ('geotiff_out', False), ('test', False), ('exclude_border', False), ('extmask', None), ('exclude_splitshp', False), ('verbose', False), ('format', ['statistics'])]) +INFO:root:checking the projection of the inputfile now +INFO:root:input shapefile had other than EPSG 4326 but was reprojected and works now +INFO:root:checking the projection of the inputfile now +INFO:root:input shapefile had other than EPSG 4326 but was reprojected and works now +INFO:root:checking the projection of the inputfile now +INFO:root:input shapefile had other than EPSG 4326 but was reprojected and works now +INFO:root:number of usable cores for shapesplitting is 2 +INFO:root:splitted shapefiles now exist +INFO:root:deleted splitted worldtiles +INFO:root:Imagepath is data_collection/S2B_MSIL2A_20200626T095029_N0214_R079_T34VFN_20200626T123234.SAFE/./GRANULE/L2A_T34VFN_A017265_20200626T095032/IMG_DATA +INFO:root:Tile is 34VFN +INFO:root:Date is 20200626 +INFO:root:Shape of cloudmask is (10980, 10980) +INFO:root:Checking the projection of the inputfile now +INFO:root:Reprojectcommand: ogr2ogr -t_srs EPSG:32634 EODIE_temp_shp/shapefile_34VFN_reprojected_32634.shp EODIE_temp_shp/shapefile_reprojected_4326_34VFN.shp +INFO:root:input shapefile had other than EPSG 32634 but was reprojected and works now +INFO:root:Cloudcover below 99: True +INFO:root:Data withing area of interest: True +INFO:root:stat to csv in: ./results/ndvi_20200626_34VFN_statistics.csv +INFO:root:deleted splitted shapefiles diff -r 000000000000 -r 81b0ca76435d test-data/20211218-102629.log --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/20211218-102629.log Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,19 @@ +INFO:root:All inputs for this process: dict_items([('platform', 'ls8'), ('config', {'platform': 'ls8', 'maxcloudcover': 99, 'bitmask': 1, 'tobemaskedlist': [0, 1, 2, 3, 4, 5, 9, 11, 13, 15], 'filepattern': 'LC08_L\\d.._\\d{6}_\\d{8}_\\d{8}_02_[TR][12T]$', 'bandlocation': ['.'], 'cloudfilename': 'QA_PIXEL', 'red': 'B4', 'green': 'B3', 'blue': 'B2', 'nir': 'B5', 'swir1': 'B6', 'swir2': 'B7', 'tilepattern': '[0-9]{6}', 'datepattern': '20[1-2][0-9][0-1][0-9][0-3][0-9]', 'band_designation': 'B\\d?\\d', 'quantification_value': 65535, 'B1': [30], 'B2': [30], 'B3': [30], 'B4': [30], 'B5': [30], 'B6': [30], 'B7': [30], 'QA_PIXEL': [30], 'resampling_method': 'bilinear', 'pathbuildinglist': ['*', 'bandname', '*', '.TIF'], 'pixelsize': 30, 'tileshp': './WRS2_descending/WRS2_descending', 'fieldname': 'PR', 'lookup': './postprocesses/lookuptable.txt'}), ('mydir', 'data_collection'), ('myfile', None), ('input', ['data_collection/LC08_L2SP_189017_20190628_20200827_02_T1']), ('shpbase', 'shapefile'), ('outpath', './results'), ('idname', 'ID'), ('statistics_out', True), ('array_out', False), ('indexlist', ['ndvi']), ('statistics', ['count', 'mean', 'median', 'std']), ('startdate', '20160101'), ('enddate', '20211221'), ('keep_shp', False), ('geotiff_out', False), ('test', False), ('exclude_border', False), ('extmask', None), ('exclude_splitshp', False), ('verbose', False), ('format', ['statistics'])]) +INFO:root:checking the projection of the inputfile now +INFO:root:input shapefile had other than EPSG 4326 but was reprojected and works now +INFO:root:checking the projection of the inputfile now +INFO:root:input shapefile had other than EPSG 4326 but was reprojected and works now +INFO:root:checking the projection of the inputfile now +INFO:root:input shapefile had other than EPSG 4326 but was reprojected and works now +INFO:root:number of usable cores for shapesplitting is 2 +INFO:root:splitted shapefiles now exist +INFO:root:deleted splitted worldtiles +INFO:root:Imagepath is data_collection/LC08_L2SP_189017_20190628_20200827_02_T1/. +INFO:root:Tile is 189017 +INFO:root:Date is 20190628 +INFO:root:Shape of cloudmask is (40, 27) +INFO:root:Checking the projection of the inputfile now +INFO:root:Reprojectcommand: ogr2ogr -t_srs EPSG:32635 EODIE_temp_shp/shapefile_189017_reprojected_32635.shp EODIE_temp_shp/shapefile_reprojected_4326_189017.shp +INFO:root:input shapefile had other than EPSG 32635 but was reprojected and works now +INFO:root:stat to csv in: ./results/ndvi_20190628_189017_statistics.csv +INFO:root:deleted splitted shapefiles diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/WRS2_descending.html Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,9 @@ +Shapefile Galaxy Composite Dataset

+

This composite dataset is composed of the following files:

diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending/WRS2_descending.dbf Binary file test-data/WRS2_descending/WRS2_descending.dbf has changed diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending/WRS2_descending.prj --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/WRS2_descending/WRS2_descending.prj Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]] \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending/WRS2_descending.sbn Binary file test-data/WRS2_descending/WRS2_descending.sbn has changed diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending/WRS2_descending.sbx Binary file test-data/WRS2_descending/WRS2_descending.sbx has changed diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending/WRS2_descending.shp Binary file test-data/WRS2_descending/WRS2_descending.shp has changed diff -r 000000000000 -r 81b0ca76435d test-data/WRS2_descending/WRS2_descending.shx Binary file test-data/WRS2_descending/WRS2_descending.shx has changed diff -r 000000000000 -r 81b0ca76435d test-data/ndvi_20190628_189017_statistics.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/ndvi_20190628_189017_statistics.csv Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,4 @@ +id count mean median std +0 32 0.384 0.387 0.033 +1 81 0.304 0.295 0.078 +2 48 0.424 0.432 0.031 diff -r 000000000000 -r 81b0ca76435d test-data/ndvi_20200626_34VFN_statistics.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/ndvi_20200626_34VFN_statistics.csv Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,4 @@ +id count mean median std +0 204 0.321 0.302 0.091 +1 551 0.293 0.294 0.088 +2 335 0.661 0.686 0.091 diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/sentinel2_tiles_world.html Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,7 @@ +Shapefile Galaxy Composite Dataset

+

This composite dataset is composed of the following files:

diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/COPYING.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/sentinel2_tiles_world/COPYING.txt Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,3 @@ +© Copernicus Sentinel data 2016 downloaded from https://scihub.copernicus.eu/ + +Read the TERMS AND CONDITIONS \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/TERMS AND CONDITIONS.pdf Binary file test-data/sentinel2_tiles_world/TERMS AND CONDITIONS.pdf has changed diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/readme.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/sentinel2_tiles_world/readme.txt Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,3 @@ +© Copernicus Sentinel data 2016 downloaded from https://scihub.copernicus.eu/ + +Sentinel-2 tiling grid adapted from the kml file downloaded from ESA (https://sentinel.esa.int/web/sentinel/missions/sentinel-2/data-products) at the link https://sentinel.esa.int/documents/247904/1955685/S2A_OPER_GIP_TILPAR_MPC__20151209T095117_V20150622T000000_21000101T000000_B00.kml \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/sentinel2_tiles_world.dbf Binary file test-data/sentinel2_tiles_world/sentinel2_tiles_world.dbf has changed diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/sentinel2_tiles_world.prj --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/sentinel2_tiles_world/sentinel2_tiles_world.prj Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,1 @@ +GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137,298.257223563]],PRIMEM["Greenwich",0],UNIT["Degree",0.017453292519943295]] \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/sentinel2_tiles_world.qpj --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/sentinel2_tiles_world/sentinel2_tiles_world.qpj Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,1 @@ +GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0,AUTHORITY["EPSG","8901"]],UNIT["degree",0.0174532925199433,AUTHORITY["EPSG","9122"]],AUTHORITY["EPSG","4326"]] diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/sentinel2_tiles_world.shp Binary file test-data/sentinel2_tiles_world/sentinel2_tiles_world.shp has changed diff -r 000000000000 -r 81b0ca76435d test-data/sentinel2_tiles_world/sentinel2_tiles_world.shx Binary file test-data/sentinel2_tiles_world/sentinel2_tiles_world.shx has changed diff -r 000000000000 -r 81b0ca76435d test-data/smaller_area_20100401.tif Binary file test-data/smaller_area_20100401.tif has changed diff -r 000000000000 -r 81b0ca76435d test-data/test_parcels_32635.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test_parcels_32635.html Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,7 @@ +Shapefile Galaxy Composite Dataset

+

This composite dataset is composed of the following files:

diff -r 000000000000 -r 81b0ca76435d test-data/test_parcels_32635/test_parcels_32635.cpg --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test_parcels_32635/test_parcels_32635.cpg Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,1 @@ +UTF-8 \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/test_parcels_32635/test_parcels_32635.dbf Binary file test-data/test_parcels_32635/test_parcels_32635.dbf has changed diff -r 000000000000 -r 81b0ca76435d test-data/test_parcels_32635/test_parcels_32635.prj --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test_parcels_32635/test_parcels_32635.prj Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,1 @@ +PROJCS["WGS_1984_UTM_Zone_35N",GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",27.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/test_parcels_32635/test_parcels_32635.shp Binary file test-data/test_parcels_32635/test_parcels_32635.shp has changed diff -r 000000000000 -r 81b0ca76435d test-data/test_parcels_32635/test_parcels_32635.shx Binary file test-data/test_parcels_32635/test_parcels_32635.shx has changed diff -r 000000000000 -r 81b0ca76435d test-data/test_polygons.html --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test_polygons.html Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,7 @@ +Shapefile Galaxy Composite Dataset

+

This composite dataset is composed of the following files:

diff -r 000000000000 -r 81b0ca76435d test-data/test_polygons/test_polygons.dbf Binary file test-data/test_polygons/test_polygons.dbf has changed diff -r 000000000000 -r 81b0ca76435d test-data/test_polygons/test_polygons.prj --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/test_polygons/test_polygons.prj Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,1 @@ +PROJCS["EUREF_FIN_TM35FIN",GEOGCS["GCS_ETRS_1989",DATUM["D_ETRS_1989",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Transverse_Mercator"],PARAMETER["False_Easting",500000.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",27.0],PARAMETER["Scale_Factor",0.9996],PARAMETER["Latitude_Of_Origin",0.0],UNIT["Meter",1.0]] \ No newline at end of file diff -r 000000000000 -r 81b0ca76435d test-data/test_polygons/test_polygons.shp Binary file test-data/test_polygons/test_polygons.shp has changed diff -r 000000000000 -r 81b0ca76435d test-data/test_polygons/test_polygons.shx Binary file test-data/test_polygons/test_polygons.shx has changed diff -r 000000000000 -r 81b0ca76435d test-data/testrgb_20100401__statistics.csv --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/testrgb_20100401__statistics.csv Thu Dec 30 15:24:09 2021 +0000 @@ -0,0 +1,4 @@ +id count mean percentile_10 +3 20508 154.641 110.000 +1 12049 181.347 147.000 +2 4508 89.314 32.000