Mercurial > repos > climate > cads
changeset 0:b768e517a7d0 draft default tip
"planemo upload for repository https://github.com/NordicESMhub/galaxy-tools/tree/master/tools/cads commit 778688ee8d14e22b00b648858ad9bd1c2f46b1e0"
| author | climate |
|---|---|
| date | Sat, 19 Jun 2021 18:35:50 +0000 |
| parents | |
| children | |
| files | README.md cads.sh cads.xml cads_retrieve.py test-data/output.nc test-data/req-timeseries.txt test-data/req.txt test-data/req_out.txt test-data/timeseries.nc test-data/timeseries_out.txt |
| diffstat | 10 files changed, 257 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.md Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,25 @@ + +# Copernicus Atmosphere Data Store (ADS) + +This tool allows you to retrieve [Copernicus Atmosphere Data Store (ADS)](https://ads.atmosphere.copernicus.eu/cdsapp#!/home). + +- Any user willing to use this tool needs to [create a new account](https://ads.atmosphere.copernicus.eu/user/register?destination=/). +- Compose your request directly on ADS and copy/paste it in the input field "Request" +- Users need to add their ADS API Key to Galaxy (see below). Documentation on where to get the ADS API key can be found [here](https://ads.atmosphere.copernicus.eu/api-how-to). +- Be aware that for being able to download dataset from ADS, users also need to agree to their term of use (Licence to use Copernicus Products) on the ADS website. + +## Set up user credentials on Galaxy + +To enable users to set their credentials and provide their ADS API Key for this tool, +make sure the file `config/user_preferences_extra.yml` has the following section: + +``` + cads_account: + description: Your Copernicus ADS API Key (Copernicus Atmosphere Data Store API Key) + inputs: + - name: cads_apikey + label: Copernicus ADS API Key + type: password + required: True +``` +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cads.sh Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +if [ -f download.tar.gz ]; then + tar zxvf download.tar.gz + cat *.grib > tmp.grib + cdo setgridtype,regular tmp.grib tmpg.grib + cdo -f nc -t ecmwf copy tmpg.grib tmp.nc +elif [ -f download.zip ]; then + unzip download.zip + cat *.grib > tmp.grib + cdo setgridtype,regular tmp.grib tmpg.grib + cdo -f nc -t ecmwf copy tmpg.grib tmp.nc +elif [ -f download.grib ]; then + cdo -f nc -t ecmwf copy download.grib tmp.nc +elif [ -f download.nc ]; then + mv download.nc tmp.nc +else + echo "No data found! Check your request and/or credentials." +fi
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cads.xml Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,111 @@ +<tool id="cads" name="Copernicus Atmosphere Data Store" version="0.1.0" profile="20.05"> + <description>for retrieving data from the Atmosphere Monitoring Service</description> + <requirements> + <requirement type="package" version="3">python</requirement> + <requirement type="package" version="0.5.1">cdsapi</requirement> + <requirement type="package" version="1.9.9">cdo</requirement> + <requirement type="package" version="1.34">tar</requirement> + <requirement type="package" version="6.0">unzip</requirement> + </requirements> + <command detect_errors="exit_code"><![CDATA[ + export HOME=`pwd` && + #set $cads_apikey = $__user__.extra_preferences.get('cads_account|cads_apikey', "") + #if $cads_apikey == '' + echo 'Error. Set your CADS credentials via: User -> Preferences -> Manage Information' && + echo 'Will try to use tool adsapirc file (used for testing)' && + cp '$__tool_directory__/adsapirc.cfg' .cdsapirc && + #else + cp '$cads_key_file' .cdsapirc && + #end if + + python3 '$__tool_directory__/cads_retrieve.py' + #if str($is_file.has_req).strip() == 'yes' + --request '$is_file.api_req_file' + #else + --request '$req_from_paste' + #end if + --output request.txt && + bash '$__tool_directory__/cads.sh' && + echo "Data retrieval from Copernicus Atmosphere Data Store is done" + ]]></command> + <configfiles> + <configfile name="cads_key_file"><![CDATA[ +#set $cads_apikey = $__user__.extra_preferences.get('cads_account|cads_apikey', "") + url: https://ads.atmosphere.copernicus.eu/api/v2 + key: $cads_apikey + ]]></configfile> + <configfile name="req_from_paste"><![CDATA[ + #if str($is_file.has_req).strip() == 'no' +$is_file.api_req_text + #end if + ]]></configfile> + </configfiles> + <inputs> + <conditional name="is_file"> + <param name="has_req" type="select" label="Provide CADS request"> + <option value="no">paste as text</option> + <option value="yes" selected="true">give a filename</option> + </param> + <when value="yes"> + <param name="api_req_file" type="data" label="API Request filename" format="txt"/> + </when> + <when value="no"> + <param name="api_req_text" type="text" label="Paste API Request" area="true" /> + </when> + </conditional> + </inputs> + <outputs> + <data name="request" format="txt" from_work_dir="request.txt"/> + <data name="ofilename" format="netcdf" from_work_dir="tmp.nc"/> + </outputs> + <tests> + <test> + <conditional name="is_file"> + <param name="has_req" value="yes" /> + <param name="api_req_file" ftype="txt" value="req.txt" /> + </conditional> + <output name="request" value="req_out.txt" compare="sim_size" delta="10"/> + <output name="ofilename" ftype="netcdf" value="output.nc" compare="sim_size" delta="100"/> + </test> + <test> + <conditional name="is_file"> + <param name="has_req" value="yes" /> + <param name="api_req_file" ftype="txt" value="req-timeseries.txt" /> + </conditional> + <output name="request" value="timeseries_out.txt" compare="sim_size" delta="10"/> + <output name="ofilename" ftype="netcdf" value="timeseries.nc" compare="sim_size" delta="100"/> + </test> + </tests> + <help><![CDATA[ + +**Copernicus Atmosphere Data Store (ADS)** +======================================================================================================= + +This tool is a wrapper to retrieve data from the Copernicus Atmosphere Data Store. + +- It allows to retrieve data from the Copernicus Atmosphere Monitoring Service. +- Any user willing to use this tool needs to `create a new account <https://ads.atmosphere.copernicus.eu/user/register?destination=/api-how-to|>`_. +- Set your CADS API Key via: User -> Preferences -> Manage Information" +- Documentation on where to get the CDS API key can be found `here <https://cds.climate.copernicus.eu/api-how-to>`_. +- Compose your request directly on Copernicus Atmosphere Data Store and copy/paste it in the input field "Request" or save it in a file. +- Be aware that for being able to download dataset from ADS, users also need to agree to their term of use (Licence to use Copernicus Products) on the ADS website. + +License: +~~~~~~~~ + +Generated using Copernicus Atmosphere Monitoring Service information [2021] +Neither the European Commission nor ECMWF is responsible for any use +that may be made of the Copernicus information or data it contains. + ]]></help> + <citations> + </citations> + <edam_topics> + <edam_topic>topic_3855</edam_topic> + <edam_topic>topic_3318</edam_topic> + </edam_topics> + <edam_operations> + <edam_operation>operation_2422</edam_operation> + <edam_operation>operation_3357</edam_operation> + <edam_operation>operation_0335</edam_operation> + </edam_operations> +</tool>
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cads_retrieve.py Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,56 @@ +import argparse +import ast +from os import environ, path + +import cdsapi + +parser = argparse.ArgumentParser() +parser.add_argument("-i", "--request", type=str, help="input API request") +parser.add_argument("-o", "--output", type=str, help="output API request") +args = parser.parse_args() + +mapped_chars = { + '>': '__gt__', + '<': '__lt__', + "'": '__sq__', + '"': '__dq__', + '[': '__ob__', + ']': '__cb__', + '{': '__oc__', + '}': '__cc__', + '@': '__at__', + '#': '__pd__', + "": '__cn__' +} + +with open(args.request) as f: + req = f.read() + + # Unsanitize labels (element_identifiers are always sanitized by Galaxy) + for key, value in mapped_chars.items(): + req = req.replace(value, key) + +print("req = ", req) +c3s_type = req.split('c.retrieve')[1].split('(')[1].split(',')[0].strip(' "\'\t\r\n') + +c3s_req = '{' + req.split('{', 1)[1].rsplit('}', 1)[0].replace('\n', '') + '}' +c3s_req_dict = ast.literal_eval(c3s_req) + +c3s_output = req.rsplit('}', 1)[1].split(',')[1].split(')')[0].strip(' "\'\t\r\n') + +with open(args.output, "w") as f: + f.write(f'dataset to retrieve: {c3s_type}\nrequest: {c3s_req}\noutput filename: {c3s_output}') + +print("start retrieving data...") + +cdapi_file = path.join(environ.get('HOME'), '.cdsapirc') + +if path.isfile(cdapi_file): + c = cdsapi.Client() + + c.retrieve( + c3s_type, + c3s_req_dict, + c3s_output) + + print("data retrieval successful")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/req-timeseries.txt Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,19 @@ +import cdsapi + +c = cdsapi.Client() + +c.retrieve( + 'cams-solar-radiation-timeseries', + { + 'sky_type': 'clear', + 'location': { + 'latitude': 60, + 'longitude': 11, + }, + 'altitude': '-999.', + 'date': '2021-06-17/2021-06-17', + 'time_step': '15minute', + 'time_reference': 'universal_time', + 'format': 'netcdf', + }, + 'download.nc')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/req.txt Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,21 @@ +import cdsapi + +c = cdsapi.Client() + +c.retrieve( + 'cams-europe-air-quality-forecasts', + { + 'variable': 'nitrogen_dioxide', + 'model': 'ensemble', + 'level': '0', + 'date': '2021-06-16/2021-06-16', + 'type': 'analysis', + 'time': '00:00', + 'leadtime_hour': '0', + 'format': 'netcdf', + 'area': [ + 59.96, 10.66, 59.87, + 10.9, + ], + }, + 'download.nc')
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/req_out.txt Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,3 @@ +dataset to retrieve: cams-europe-air-quality-forecasts +request: { 'variable': 'nitrogen_dioxide', 'model': 'ensemble', 'level': '0', 'date': '2021-06-16/2021-06-16', 'type': 'analysis', 'time': '00:00', 'leadtime_hour': '0', 'format': 'netcdf', 'area': [ 59.96, 10.66, 59.87, 10.9, ], } +output filename: download.nc \ No newline at end of file
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test-data/timeseries_out.txt Sat Jun 19 18:35:50 2021 +0000 @@ -0,0 +1,3 @@ +dataset to retrieve: cams-solar-radiation-timeseries +request: { 'sky_type': 'clear', 'location': { 'latitude': 60, 'longitude': 11, }, 'altitude': '-999.', 'date': '2021-06-17/2021-06-17', 'time_step': '15minute', 'time_reference': 'universal_time', 'format': 'netcdf', } +output filename: download.nc \ No newline at end of file