Mercurial > repos > ecology > hirondelle_crim
comparison ogc_api_processes_wrapper.R @ 1:52baddd15640 draft default tip
planemo upload for repository https://github.com/galaxyecology/tools-ecology/tree/master/tools/ogc_api_processes_wrapper commit 8b4b58222af2c77abd41dd8f17862a24ca7d3381
author | ecology |
---|---|
date | Fri, 06 Sep 2024 10:30:30 +0000 |
parents | afede0916f0a |
children |
comparison
equal
deleted
inserted
replaced
0:afede0916f0a | 1:52baddd15640 |
---|---|
2 library("jsonlite") | 2 library("jsonlite") |
3 library("getopt") | 3 library("getopt") |
4 | 4 |
5 cat("start generic wrapper service \n") | 5 cat("start generic wrapper service \n") |
6 | 6 |
7 getParameters <- function(){ | 7 remove_null_values <- function(x) { |
8 con <- file("inputs.json", "r") | 8 # Check if the input is a list |
9 lines <- readLines(con) | 9 if (is.list(x)) { |
10 close(con) | 10 # Remove NULL values and apply the function recursively to sublists |
11 | 11 x <- lapply(x, remove_null_values) |
12 json_string <- paste(lines, collapse = "\n") | 12 x <- x[!sapply(x, is.null)] |
13 json_data <- fromJSON(json_string) | 13 } |
14 return(json_data$conditional_process) | 14 return(x) |
15 } | |
16 | |
17 getParameters <- function() { | |
18 con <- file("inputs.json", "r") | |
19 lines <- readLines(con) | |
20 close(con) | |
21 | |
22 json_string <- paste(lines, collapse = "\n") | |
23 json_data <- fromJSON(json_string) | |
24 | |
25 # Remove NULL values from json_data | |
26 cleaned_json_data <- remove_null_values(json_data) | |
27 return(cleaned_json_data$conditional_process) | |
15 } | 28 } |
16 | 29 |
17 parseResponseBody <- function(body) { | 30 parseResponseBody <- function(body) { |
18 hex <- c(body) | 31 hex <- c(body) |
19 intValues <- as.integer(hex) | 32 intValues <- as.integer(hex) |
22 jsonObject <- jsonlite::fromJSON(readableOutput) | 35 jsonObject <- jsonlite::fromJSON(readableOutput) |
23 return(jsonObject) | 36 return(jsonObject) |
24 } | 37 } |
25 | 38 |
26 getOutputs <- function(inputs, output, server) { | 39 getOutputs <- function(inputs, output, server) { |
27 url <- paste(paste(server, "/processes/", sep = ""), inputs$select_process, sep = "") | 40 url <- |
28 request <- request(url) | 41 paste(paste(server, "/processes/", sep = ""), |
29 response <- req_perform(request) | 42 inputs$select_process, |
30 responseBody <- parseResponseBody(response$body) | 43 sep = "") |
31 outputs <- list() | 44 request <- request(url) |
32 | 45 response <- req_perform(request) |
33 for (x in 1:length(responseBody$outputs)) { | 46 responseBody <- parseResponseBody(response$body) |
34 outputformatName <- paste(names(responseBody$outputs[x]), "_outformat", sep="") | 47 outputs <- list() |
35 output_item <- list() | 48 |
36 | 49 for (x in 1:length(responseBody$outputs)) { |
37 for (p in names(inputs)) { | 50 outputformatName <- |
38 if(p == outputformatName){ | 51 paste(names(responseBody$outputs[x]), "_outformat", sep = "") |
39 format <- list("mediaType" = inputs[[outputformatName]]) | 52 output_item <- list() |
40 output_item$format <- format | 53 |
41 } | 54 for (p in names(inputs)) { |
42 } | 55 if (p == outputformatName) { |
43 output_item$transmissionMode <- "reference" | 56 format <- list("mediaType" = inputs[[outputformatName]]) |
44 outputs[[x]] <- output_item | 57 output_item$format <- format |
45 } | 58 } |
46 | 59 } |
47 names(outputs) <- names(responseBody$outputs) | 60 output_item$transmissionMode <- "reference" |
48 return(outputs) | 61 outputs[[x]] <- output_item |
62 } | |
63 | |
64 names(outputs) <- names(responseBody$outputs) | |
65 return(outputs) | |
49 } | 66 } |
50 | 67 |
51 executeProcess <- function(url, process, requestBodyData, cookie) { | 68 executeProcess <- function(url, process, requestBodyData, cookie) { |
52 url <- paste(paste(paste(url, "processes/", sep = ""), process, sep = ""), "/execution", sep = "") | 69 url <- |
53 requestBodyData$inputs$cookie <- NULL | 70 paste(paste(paste(url, "processes/", sep = ""), process, sep = ""), "/execution", sep = "") |
54 requestBodyData$inputs$select_process <- NULL | 71 requestBodyData$inputs$cookie <- NULL |
55 | 72 requestBodyData$inputs$select_process <- NULL |
56 requestBodyData$inputs$s3_access_key <- requestBodyData$inputs$user_credentials$s3_access_key | 73 |
57 requestBodyData$inputs$s3_secret_key <- requestBodyData$inputs$user_credentials$s3_secret_key | 74 requestBodyData$inputs$s3_access_key <- |
58 requestBodyData$inputs$user_credentials <- NULL | 75 requestBodyData$inputs$user_credentials$s3_access_key |
59 | 76 requestBodyData$inputs$s3_secret_key <- |
60 body <- list() | 77 requestBodyData$inputs$user_credentials$s3_secret_key |
61 body$inputs <- requestBodyData$inputs | 78 requestBodyData$inputs$user_credentials <- NULL |
62 body$mode <- "async" | 79 if (process == "plot-image") { |
63 body$response <- "document" | 80 tmp <- requestBodyData$inputs$color_scale |
64 | 81 color_scale <- gsub("__ob__", "[", tmp) |
65 response <- request(url) %>% | 82 color_scale <- gsub("__cb__", "]", color_scale) |
66 req_headers( | 83 requestBodyData$inputs$color_scale <- color_scale |
67 "Accept" = "application/json", | 84 #print(requestBodyData$inputs$color_scale) |
68 "Content-Type" = "application/json", | 85 } |
69 "Cookie" = cookie | 86 if (process == "calculate-band") { |
70 ) %>% | 87 requestBodyData$inputs$name <- "output" |
71 req_body_json(body) %>% | 88 } |
72 req_perform() | 89 if (process == "reproject-image") { |
73 | 90 requestBodyData$inputs$output_name <- "output" |
74 cat("\n Process executed") | 91 } |
75 cat("\n status: ", response$status_code) | 92 #requestBodyData$inputs$input_image$href <- "https://hirondelle.crim.ca/wpsoutputs/weaver/public/test-data/S2A_MSIL2A_20190701T110621_N0500_R137_T29SPC_20230604T023542_turbidity.tiff" |
76 cat("\n jobID: ", parseResponseBody(response$body)$jobID, "\n") | 93 |
77 | 94 body <- list() |
78 jobID <- parseResponseBody(response$body)$jobID | 95 body$inputs <- requestBodyData$inputs |
79 | 96 #print(body$inputs) |
80 return(jobID) | 97 body$mode <- "async" |
98 body$response <- "document" | |
99 #print(body$inputs) | |
100 | |
101 response <- request(url) %>% | |
102 req_headers("Accept" = "application/json", | |
103 "Content-Type" = "application/json", | |
104 "Cookie" = cookie) %>% | |
105 req_body_json(body) %>% | |
106 req_perform() | |
107 | |
108 cat("\n Process executed") | |
109 cat("\n status: ", response$status_code) | |
110 cat("\n jobID: ", parseResponseBody(response$body)$jobID, "\n") | |
111 | |
112 jobID <- parseResponseBody(response$body)$jobID | |
113 | |
114 return(jobID) | |
81 } | 115 } |
82 | 116 |
83 checkJobStatus <- function(server, process, jobID, cookie) { | 117 checkJobStatus <- function(server, process, jobID, cookie) { |
84 url <- paste0(server, "processes/", process, "/jobs/", jobID) | 118 url <- paste0(server, "processes/", process, "/jobs/", jobID) |
85 response <- request(url) %>% | 119 response <- request(url) %>% |
86 req_headers( | 120 req_headers("Cookie" = cookie) %>% |
87 "Cookie" = cookie | |
88 ) %>% | |
89 req_perform() | 121 req_perform() |
90 jobStatus <- parseResponseBody(response$body)$status | 122 jobStatus <- parseResponseBody(response$body)$status |
91 jobProgress <- parseResponseBody(response$body)$progress | 123 jobProgress <- parseResponseBody(response$body)$progress |
92 return(jobStatus) | 124 return(jobStatus) |
93 } | 125 } |
94 | 126 |
95 getStatusCode <- function(server, process, jobID, cookie) { | 127 getStatusCode <- function(server, process, jobID, cookie) { |
96 url <- paste0(server, "processes/", process, "/jobs/", jobID) | 128 url <- paste0(server, "processes/", process, "/jobs/", jobID) |
97 response <- request(url) %>% | 129 response <- request(url) %>% |
98 req_headers( | 130 req_headers("Cookie" = cookie) %>% |
99 "Cookie" = cookie | |
100 ) %>% | |
101 req_perform() | 131 req_perform() |
102 status_code <- response$status_code | 132 status_code <- response$status_code |
103 return(status_code) | 133 return(status_code) |
104 } | 134 } |
105 | 135 |
106 getResult <- function (server, process, jobID, cookie) { | 136 getResult <- function (server, process, jobID, cookie) { |
107 response <- request(paste0(server, "processes/", process, "/jobs/", jobID, "/results")) %>% | 137 response <- |
108 req_headers( | 138 request(paste0(server, "processes/", process, "/jobs/", jobID, "/results")) %>% |
109 "Cookie" = cookie | 139 req_headers("Cookie" = cookie) %>% |
110 ) %>% | |
111 req_perform() | 140 req_perform() |
112 return(response) | 141 return(response) |
113 } | 142 } |
114 | 143 |
115 retrieveResults <- function(server, process, jobID, outputData, cookie) { | 144 retrieveResults <- |
145 function(server, process, jobID, outputData, cookie) { | |
116 status_code <- getStatusCode(server, process, jobID, cookie) | 146 status_code <- getStatusCode(server, process, jobID, cookie) |
117 if(status_code == 200){ | 147 if (status_code == 200) { |
118 status <- "running" | 148 status <- "running" |
119 while(status == "running"){ | 149 while (status == "running") { |
120 jobStatus <- checkJobStatus(server, process, jobID, cookie) | 150 jobStatus <- checkJobStatus(server, process, jobID, cookie) |
121 print(jobStatus) | 151 print(jobStatus) |
122 if (jobStatus == "succeeded") { | 152 if (jobStatus == "succeeded") { |
123 status <- jobStatus | 153 status <- jobStatus |
124 result <- getResult(server, process, jobID, cookie) | 154 result <- getResult(server, process, jobID, cookie) |
125 if (result$status_code == 200) { | 155 if (result$status_code == 200) { |
126 resultBody <- parseResponseBody(result$body) | 156 resultBody <- parseResponseBody(result$body) |
127 urls <- unname(unlist(lapply(resultBody, function(x) x$href))) | 157 #print(resultBody) |
128 urls_with_newline <- paste(urls, collapse = "\n") | 158 if (process == "select-products-sentinel2") { |
129 con <- file(outputData, "w") | 159 urls <- unname(unlist(lapply(resultBody, function(x) |
130 writeLines(urls_with_newline, con = con) | 160 x$value))) |
131 close(con) | 161 } else if (process == "download-band-sentinel2-product-safe" || |
132 } | 162 process == "calculate-band" || |
133 } else if (jobStatus == "failed") { | 163 process == "plot-image" || process == "reproject-image") { |
134 status <- jobStatus | 164 urls <- unname(unlist(lapply(resultBody, function(x) |
165 x$href))) | |
135 } | 166 } |
167 urls_with_newline <- paste(urls, collapse = "\n") | |
168 con <- file(outputData, "w") | |
169 writeLines(urls_with_newline, con = con) | |
170 close(con) | |
171 } | |
172 } else if (jobStatus == "failed") { | |
173 status <- jobStatus | |
174 } | |
136 Sys.sleep(3) | 175 Sys.sleep(3) |
137 } | 176 } |
138 cat("\n done \n") | 177 cat("\n done \n") |
139 } else if (status_code1 == 400) { | 178 } else if (status_code1 == 400) { |
140 print("A query parameter has an invalid value.") | 179 print("A query parameter has an invalid value.") |
141 } else if (status_code1 == 404) { | 180 } else if (status_code1 == 404) { |
142 print("The requested URI was not found.") | 181 print("The requested URI was not found.") |
143 } else if (status_code1 == 500) { | 182 } else if (status_code1 == 500) { |
144 print("The requested URI was not found.") | 183 print("The requested URI was not found.") |
145 } else { | 184 } else { |
146 print(paste("HTTP", status_code1, "Error:", resp1$status_message)) | 185 print(paste("HTTP", status_code1, "Error:", resp1$status_message)) |
147 } | 186 } |
148 } | 187 } |
149 | 188 |
150 is_url <- function(x) { | 189 is_url <- function(x) { |
151 grepl("^https?://", x) | 190 grepl("^https?://", x) |
152 } | 191 } |
153 | 192 |
154 server <- "https://hirondelle.crim.ca/weaver/" | 193 server <- "https://hirondelle.crim.ca/weaver/" |
155 | 194 |
156 print("--> Retrieve parameters") | 195 print("--> Retrieve parameters") |
157 inputParameters <- getParameters() | 196 inputParameters <- getParameters() |
197 #print(inputParameters) | |
158 print("--> Parameters retrieved") | 198 print("--> Parameters retrieved") |
159 | 199 |
160 args <- commandArgs(trailingOnly = TRUE) | 200 args <- commandArgs(trailingOnly = TRUE) |
161 outputLocation <- args[2] | 201 outputLocation <- args[2] |
162 | 202 |
165 print("--> Outputs retrieved") | 205 print("--> Outputs retrieved") |
166 | 206 |
167 print("--> Parse inputs") | 207 print("--> Parse inputs") |
168 convertedKeys <- c() | 208 convertedKeys <- c() |
169 for (key in names(inputParameters)) { | 209 for (key in names(inputParameters)) { |
170 if (is.character(inputParameters[[key]]) && (endsWith(inputParameters[[key]], ".dat") || endsWith(inputParameters[[key]], ".txt"))) { | 210 if (is.character(inputParameters[[key]]) && |
211 (endsWith(inputParameters[[key]], ".dat") || | |
212 endsWith(inputParameters[[key]], ".txt"))) { | |
171 con <- file(inputParameters[[key]], "r") | 213 con <- file(inputParameters[[key]], "r") |
172 url_list <- list() | 214 url_list <- list() |
173 #while (length(line <- readLines(con, n = 1)) > 0) { | 215 #while (length(line <- readLines(con, n = 1)) > 0) { |
174 # if (is_url(line)) { | 216 # if (is_url(line)) { |
175 # url_list <- c(url_list, list(list(href = trimws(line)))) | 217 # url_list <- c(url_list, list(list(href = trimws(line)))) |
176 # } | 218 # } |
177 #} | 219 #} |
178 con <- file(inputParameters[[key]], "r") | 220 con <- file(inputParameters[[key]], "r") |
179 lines <- readLines(con) | 221 lines <- readLines(con) |
222 print("--------------------------------------------------------------------1") | |
223 print(length(lines)) | |
180 close(con) | 224 close(con) |
181 json_string <- paste(lines, collapse = "\n") | 225 if (!length(lines) > 1 && endsWith(lines, ".jp2") && startsWith(lines, "https")) { |
182 json_data <- fromJSON(json_string) | 226 print("--------------------------------------------------------------------2") |
183 | 227 tmp <- list() |
184 inputParameters[[key]] <- json_data | 228 tmp$href <- lines |
229 tmp$type <- "image/jp2" | |
230 inputParameters[[key]] <- tmp | |
231 } | |
232 else if (!length(lines) > 1 && endsWith(lines, ".SAFE") && startsWith(lines, "s3:")) { | |
233 print("--------------------------------------------------------------------3") | |
234 json_string <- paste(lines, collapse = "\n") | |
235 inputParameters[[key]] <- json_string | |
236 } else if (inputParameters$select_process == "plot-image" || | |
237 inputParameters$select_process == "reproject-image") { | |
238 print("--------------------------------------------------------------------4") | |
239 tmp <- list() | |
240 tmp$href <- lines | |
241 tmp$type <- "image/tiff; application=geotiff" | |
242 if (inputParameters$select_process == "reproject-image") { | |
243 tmp$type <- "image/tiff; subtype=geotiff" | |
244 } | |
245 inputParameters[[key]] <- tmp | |
246 } else { | |
247 print("-----------------------------------5") | |
248 json_string <- paste(lines, collapse = "\n") | |
249 json_data <- fromJSON(json_string) | |
250 inputParameters[[key]] <- json_data | |
251 } | |
185 convertedKeys <- append(convertedKeys, key) | 252 convertedKeys <- append(convertedKeys, key) |
186 } | 253 } |
187 else if (grepl("_Array_", key)) { | 254 else if (grepl("_Array_", key)) { |
188 keyParts <- strsplit(key, split = "_")[[1]] | 255 keyParts <- strsplit(key, split = "_")[[1]] |
189 type <- keyParts[length(keyParts)] | 256 type <- keyParts[length(keyParts)] |
190 values <- inputParameters[[key]] | 257 values <- inputParameters[[key]] |
191 value_list <- strsplit(values, split = ",") | 258 value_list <- strsplit(values, split = ",") |
192 | |
193 convertedValues <- c() | 259 convertedValues <- c() |
194 | 260 |
195 for (value in value_list) { | 261 for (value in value_list) { |
196 if(type == "integer") { | 262 if (type == "integer") { |
197 value <- as.integer(value) | 263 value <- as.integer(value) |
198 } else if (type == "numeric") { | 264 } else if (type == "numeric") { |
199 value <- as.numeric(balue) | 265 value <- as.numeric(balue) |
200 } else if (type == "character") { | 266 } else if (type == "character") { |
201 value <- as.character(value) | 267 value <- as.character(value) |
202 } | 268 } |
203 convertedValues <- append(convertedValues, value) | 269 convertedValues <- append(convertedValues, value) |
204 | 270 |
205 convertedKey <- "" | 271 convertedKey <- "" |
206 for (part in keyParts) { | 272 for (part in keyParts) { |
207 if(part == "Array") { | 273 if (part == "Array") { |
208 break | 274 break |
209 } | 275 } |
210 convertedKey <- paste(convertedKey, paste(part, "_", sep=""), sep="") | 276 convertedKey <- |
211 } | 277 paste(convertedKey, paste(part, "_", sep = ""), sep = "") |
212 convertedKey <- substr(convertedKey, 1, nchar(convertedKey)-1) | 278 } |
213 } | 279 convertedKey <- substr(convertedKey, 1, nchar(convertedKey) - 1) |
214 | 280 } |
281 | |
215 inputParameters[[key]] <- convertedValues | 282 inputParameters[[key]] <- convertedValues |
283 #print("-------------------------") | |
284 #print(convertedValues) | |
285 #print("-------------------------") | |
216 convertedKeys <- append(convertedKeys, convertedKey) | 286 convertedKeys <- append(convertedKeys, convertedKey) |
217 } else { | 287 } else { |
218 convertedKeys <- append(convertedKeys, key) | 288 #print("-------------------------") |
219 } | 289 #print(key) |
220 } | 290 #print(inputParameters[[key]]) |
221 | 291 if (!is.null(inputParameters[[key]])) { |
292 convertedKeys <- append(convertedKeys, key) | |
293 } | |
294 #print("-------------------------") | |
295 | |
296 } | |
297 } | |
298 #print(inputParameters) | |
222 names(inputParameters) <- convertedKeys | 299 names(inputParameters) <- convertedKeys |
300 #print(inputParameters) | |
223 print("--> Inputs parsed") | 301 print("--> Inputs parsed") |
224 | 302 |
225 print("--> Prepare process execution") | 303 print("--> Prepare process execution") |
226 jsonData <- list( | 304 jsonData <- list("inputs" = inputParameters, |
227 "inputs" = inputParameters, | 305 "outputs" = outputs) |
228 "outputs" = outputs | |
229 ) | |
230 | 306 |
231 cookie <- inputParameters$cookie | 307 cookie <- inputParameters$cookie |
232 | 308 |
233 print("--> Execute process") | 309 print("--> Execute process") |
234 jobID <- executeProcess(server, inputParameters$select_process, jsonData, cookie) | 310 jobID <- |
311 executeProcess(server, inputParameters$select_process, jsonData, cookie) | |
235 print("--> Process executed") | 312 print("--> Process executed") |
236 | 313 |
237 print("--> Retrieve results") | 314 print("--> Retrieve results") |
238 retrieveResults(server, inputParameters$select_process, jobID, outputLocation, cookie) | 315 retrieveResults(server, |
316 inputParameters$select_process, | |
317 jobID, | |
318 outputLocation, | |
319 cookie) | |
239 print("--> Results retrieved") | 320 print("--> Results retrieved") |