Mercurial > repos > rhohensinner > galaxy_irods_interface

comparison main.py @ 0:7d8912d20007 draft

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit b2a00d9c24285fef0fb131d1832ecf4c337e5038-dirty"
author rhohensinner
date Fri, 21 May 2021 09:51:30 +0000
parents
children 19c1cecdfdfd
comparison
equal deleted inserted replaced
-1:000000000000 0:7d8912d20007
1 ___author___ = "Richard Hohensinner"
2 ___created___ = "04.02.2021"
3 ___last_modified___ = "20.05.2021"
4
5 # general imports
6 import os, sys, json
7 from shutil import copyfile
8 from datetime import datetime
9
10 # irods-client imports
11 from irods.session import iRODSSession
12 from irods.models import Collection, DataObject
13 from irods.query import SpecificQuery
14
15 # Tkinter imports
16 from tkinter import messagebox, Label, Button, Entry, Listbox, Tk, PhotoImage, Grid
17
18 # global variables
19 session = None
20 file_path_list = []
21 selected_file = ""
22 session_success = False
23 selection_success = False
24 iRODSCredentials = {"host": "", "port": "", "user": "", "pw": "", "zone": ""}
25
26
27 ########################################################################################################################
28 # Main function of the iRODS tools
29 #
30 # IN:
31 # JSON Object params (argv[1])
32 # OUT:
33 #
34 ########################################################################################################################
35 def main():
36
37 # check input parameters
38 if len(sys.argv) == 2:
39 params = json.loads(sys.argv[1])
40 else:
41 raise Exception("Invalid Parameters submitted!")
42
43 tool_type = params["tool_type"]
44
45 try:
46 if params["tool_type"] != "up" and params["tool_type"] != "down":
47 raise Exception("Invalid tool-type parameter submitted!")
48 except:
49 raise Exception("No tool-type parameter submitted!")
50 is_download_call = True
51 if tool_type == "up":
52 is_download_call = False
53
54 # check params for integrity
55 result_string, params_faulty = check_params(params)
56
57 if params_faulty:
58 raise Exception(result_string)
59
60 global iRODSCredentials, session_success, selected_file, selection_success
61 iRODSCredentials["host"] = params["irods_host"]
62 iRODSCredentials["port"] = params["irods_port"]
63 iRODSCredentials["zone"] = params["irods_zone"]
64
65 # create login window
66 make_login_window()
67 # check tool settings and start tool execution
68 if session_success:
69 # initialize download tool
70 if params["tool_type"] == "down":
71
72 if params["selection_type"] == "explorer":
73 make_file_select_window()
74 else:
75 if (params["file_path"] != ""): #and ("/" in params["file_path"]):
76 selected_file = params["file_path"]
77 else:
78 raise Exception("Invalid File Path submitted!")
79
80 if selection_success or params["selection_type"] == "path":
81 params["user"] = iRODSCredentials["pw"]
82 params["password"] = iRODSCredentials["user"]
83 # start download routine
84 handle_download_call(params)
85 else:
86 raise Exception("File Selection failed (No file selected)")
87
88 # initialize upload tool
89 elif params["tool_type"] == "up":
90
91 if session_success:
92 params["user"] = iRODSCredentials["pw"]
93 params["password"] = iRODSCredentials["user"]
94 # start upload routine
95 handle_upload_call(params)
96 else:
97 raise Exception("Logging into iRODS failed")
98 else:
99 raise Exception("Logging into iRODS failed")
100 # -------------------------------------------------------------------------------------------------------------------- #
101
102
103 ########################################################################################################################
104 # Login Window class for Tkinter
105 #
106 # IN:
107 # Tk Window win
108 # OUT:
109 # (sets global variables iRODSCredentials, session and session_success)
110 #
111 ########################################################################################################################
112 class LoginWindow:
113 def __init__(self, win):
114 self.window = win
115 self.lbl1 = Label(win, text='iRODS Username:')
116 self.lbl2 = Label(win, text='iRODS Password:')
117 self.t1 = Entry(bd=3)
118 self.t2 = Entry(show="*")
119 self.b1 = Button(win, text='Login', command=self.login)
120
121 self.window.grid()
122 Grid.rowconfigure(self.window, 0, weight=1)
123 Grid.rowconfigure(self.window, 1, weight=1)
124 Grid.rowconfigure(self.window, 2, weight=1)
125 Grid.rowconfigure(self.window, 3, weight=1)
126 Grid.rowconfigure(self.window, 4, weight=1)
127 Grid.columnconfigure(self.window, 0, weight=1)
128
129 self.lbl1.grid(row=0, column=0, padx="20", pady="1", sticky="w")
130 self.t1.grid(row=1, column=0, padx="10", pady="1", sticky="nsew")
131 self.lbl2.grid(row=2, column=0, padx="20", pady="1", sticky="w")
132 self.t2.grid(row=3, column=0, padx="10", pady="1", sticky="nsew")
133 self.b1.grid(row=4, column=0, padx="50", pady="10", sticky="nsew")
134
135 def login(self):
136 global iRODSCredentials
137 user = str(self.t1.get())
138 password = str(self.t2.get())
139 if user == "" or password == "":
140 self.window.iconify()
141 messagebox.showerror("Error", "Username or Password empty!")
142 self.window.deiconify()
143 return
144 else:
145 iRODSCredentials["user"] = user
146 iRODSCredentials["pw"] = password
147
148 get_irods_session(self.window)
149 if not session_success:
150 return
151
152 self.window.destroy()
153 # -------------------------------------------------------------------------------------------------------------------- #
154
155
156 ########################################################################################################################
157 # File Selection Window class for Tkinter
158 #
159 # IN:
160 # Tk Window win
161 # OUT:
162 # (sets global variables selected_file and selection_success)
163 #
164 ########################################################################################################################
165 class FileSelectWindow:
166 def __init__(self, win):
167 global session, iRODSCredentials
168 self.session = session
169 self.window = win
170 self.b1 = Button(win, text='Select', command=self.select)
171 self.lb1 = Listbox(win)
172
173 self.window.grid()
174 Grid.rowconfigure(self.window, 0, weight=1)
175 Grid.rowconfigure(self.window, 1, weight=1)
176 Grid.columnconfigure(self.window, 0, weight=1)
177
178 self.lb1.grid(row=0, column=0, padx="20", pady="1", sticky="nswe")
179 self.b1.grid(row=1, column=0, padx="50", pady="1", sticky="ew")
180
181 coll = session.collections.get("/" + iRODSCredentials["zone"] + "/" + "home" + "/" + iRODSCredentials["user"])
182 file_list = []
183
184 self.get_files_from_collections(coll, file_list)
185
186 for counter in range(len(file_list)):
187 self.lb1.insert(counter, file_list[counter])
188
189 def get_files_from_collections(self, coll, file_list):
190 for obj in coll.data_objects:
191 file_list.append(obj.path)
192
193 for col in coll.subcollections:
194 self.get_files_from_collections(col, file_list)
195
196 def select(self):
197 global session, selected_file, selection_success
198 try:
199 selection = self.lb1.get(self.lb1.curselection())
200 except:
201 self.window.iconify()
202 messagebox.showerror("Error", "No file selected!")
203 self.window.deiconify()
204 return
205
206 selected_file = selection
207 selection_success = True
208 self.window.destroy()
209 # -------------------------------------------------------------------------------------------------------------------- #
210
211
212 ########################################################################################################################
213 # Creates an iRODS session and sets the global session variable
214 #
215 # IN:
216 # Tk Window window
217 #
218 # OUT:
219 #
220 ########################################################################################################################
221 def get_irods_session(window):
222 global iRODSCredentials
223 host = iRODSCredentials["host"]
224 port = iRODSCredentials["port"]
225 user = iRODSCredentials["user"]
226 password = iRODSCredentials["pw"]
227 zone = iRODSCredentials["zone"]
228
229 iRODSsession = get_iRODS_connection(host=host, port=port, user=user, password=password, zone=zone)
230 global session, session_success
231 try:
232 coll = iRODSsession.collections.get("/" + zone + "/" + "home" + "/" + user)
233 except Exception:
234 window.iconify()
235 messagebox.showerror("Error", "Invalid Authentification")
236 window.deiconify()
237 return
238
239 if coll:
240 session = iRODSsession
241 session_success = True
242 # -------------------------------------------------------------------------------------------------------------------- #
243
244
245 ########################################################################################################################
246 # Helper function to initialize Login Window classes and Tk windows
247 #
248 # IN:
249 #
250 # OUT:
251 #
252 ########################################################################################################################
253 def make_login_window():
254 window = Tk()
255 LoginWindow(window)
256 window.title('iRODS Login')
257 window.geometry("450x225+10+10")
258 window.minsize(450, 225)
259 window.tk.call('wm', 'iconphoto', window._w, PhotoImage(file='/home/richard/git/galaxy_irods_tools/login.png'))
260 # alternative options:
261 # window.iconphoto(False, PhotoImage(file='/path/to/ico/icon.png'))
262 # window.iconbitmap("/home/richard/git/galaxy_irods_tools/login.ico")
263 window.mainloop()
264 # -------------------------------------------------------------------------------------------------------------------- #
265
266
267 ########################################################################################################################
268 # Helper function to initialize File Selection Window classes and Tk windows
269 #
270 # IN:
271 #
272 # OUT:
273 #
274 ########################################################################################################################
275 def make_file_select_window():
276 window = Tk()
277 FileSelectWindow(window)
278 window.title('iRODS File Select')
279 window.geometry("450x225+10+10")
280 window.minsize(450, 225)
281 window.mainloop()
282 # -------------------------------------------------------------------------------------------------------------------- #
283
284
285 ########################################################################################################################
286 # Checks whether arguments are valid and returns true/false depending on params
287 #
288 # IN:
289 # Dict params
290 #
291 # OUT:
292 # String res_string
293 # Bool res_bool
294 ########################################################################################################################
295 def check_params(params):
296 res_string = ""
297 res_bool = False
298
299 try:
300 if params["irods_host"] == "":
301 res_string += "Host empty!\n"
302 res_bool = True
303 if params["irods_port"] == "":
304 res_string += "Port empty!\n"
305 res_bool = True
306 if params["irods_zone"] == "":
307 res_string += "Zone empty!\n"
308 res_bool = True
309 if params["selection_type"] == "path" and params["file_path"] == "":
310 res_string += "Missing file path!\n"
311 res_bool = True
312 except:
313 raise Exception("Invalid/Missing Parameters")
314
315 return res_string, res_bool
316 # -------------------------------------------------------------------------------------------------------------------- #
317
318
319 ########################################################################################################################
320 # Function to handle iRODS download calls
321 #
322 # IN:
323 # Dict params
324 #
325 # OUT:
326 #
327 ########################################################################################################################
328 def handle_download_call(params):
329
330 global session, selected_file
331
332 # check if /ZONE/USER/...FILE... pattern is valid
333 if len(selected_file.split("/")) < 2:
334 raise Exception("Path to file is not valid in iRODS")
335
336 file_list = []
337
338 # check if file is a directory
339 if "." not in selected_file:
340 try:
341 coll = session.collections.get(selected_file)
342 for file in coll.data_objects:
343 file_list.append(file.path)
344 except:
345 raise Exception("Invalid directory path specified!")
346 else:
347 file_list.append(selected_file)
348
349 # get registry file
350 reg_file = ""
351 for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
352 for fn in filenames:
353 if fn == "irods_galaxy_registry.xml":
354 reg_file = os.path.join(dirpath, fn)
355 if reg_file != "":
356 break
357 if reg_file != "":
358 break
359
360 # handle download for all files in file_list
361 for file in file_list:
362
363 file_to_get = file
364
365 # handle path and file name
366 name_file_to_get = file_to_get.split("/")[-1]
367 path_file_to_get = "/".join(file_to_get.split("/")[0:len(file_to_get.split("/")) - 1])
368
369 # check iRODS filesystem
370 check_iRODS_destination(session, path_file_to_get, name_file_to_get)
371
372 # get file object from iRODS
373 iRODS_file_object = session.data_objects.get(path_file_to_get + "/" + name_file_to_get)
374 input_file = iRODS_file_object.open("r+")
375 output_file = open(name_file_to_get, "wb")
376 output_file.write(input_file.read())
377
378 input_file.close()
379 output_file.close()
380
381 abs_file_path = os.path.abspath(name_file_to_get)
382
383 file_type = str(name_file_to_get.split(".")[-1])
384
385 file_content = {"uuid": None,
386 "file_type": "auto",
387 "space_to_tab": False,
388 "dbkey": "?",
389 "to_posix_lines": True,
390 "ext": file_type,
391 "path": abs_file_path,
392 "in_place": True,
393 "dataset_id": params["job_id"],
394 "type": "file",
395 "is_binary": False,
396 "link_data_only": "copy_files",
397 "name": name_file_to_get
398 }
399
400 with open("temporal.json", "w") as fileParams:
401 fileParams.write(json.dumps(file_content))
402 fileParams.close()
403
404 # load file into Galaxy by using the integrated upload tool - Preparation
405 python_command = params["galaxy_root"] + "/tools/data_source/upload.py"
406 arg1 = params["galaxy_root"]
407 arg2 = params["galaxy_datatypes"]
408 arg3 = os.path.abspath(fileParams.name)
409 arg4 = params["job_id"] + ":" + params["out_dir"] + ":" + params["out_file"]
410
411 # copy sample registry.xml to working directory
412 copyfile(reg_file, params["galaxy_datatypes"])
413
414 # activate environment for new process call and call the python upload command either both with
415 sys.path.append(params["galaxy_root"] + "/lib")
416 os.system("python -c \'import sys;sys.path.append(\"" + params["galaxy_root"] + "/lib\")\'" + " python " +
417 python_command + " " + arg1 + " " + arg2 + " " + arg3 + " " + arg4)
418
419 # close connection
420 session.cleanup()
421 # -------------------------------------------------------------------------------------------------------------------- #
422
423
424 ########################################################################################################################
425 # Function to handle iRODS upload calls
426 #
427 # IN:
428 # Dict params
429 #
430 # OUT:
431 #
432 ########################################################################################################################
433 def handle_upload_call(params):
434
435 global session, iRODSCredentials
436
437 path_to_file = params["up_file_path"]
438 name_of_file = params["up_file"]
439
440 coll_path = "/" + iRODSCredentials["zone"] + "/home/" + iRODSCredentials["user"] + "/galaxyupload"
441 try:
442 coll = session.collections.get(coll_path)
443 except:
444 coll = session.collections.create(coll_path)
445
446 now = datetime.now()
447
448 # dd/mm/YY
449 day = now.strftime("%d%m%Y")
450 time = now.strftime("%H%M%S")
451
452 coll_path = coll_path + "/" + day
453
454 try:
455 coll = session.collections.get(coll_path)
456 except:
457 coll = session.collections.create(coll_path)
458
459 irods_file_name = time + "_" + name_of_file
460 iRODS_file_object = session.data_objects.create(coll_path + "/" + irods_file_name)
461 iRODS_file_object = session.data_objects.get(coll_path + "/" + irods_file_name)
462
463 irods_file = iRODS_file_object.open("w")
464 galaxy_file = open(path_to_file, "rb")
465 content = galaxy_file.read()
466 irods_file.write(content)
467
468 # TODO can't close session without writing process finished - but reading/writing happens async.
469 # session.cleanup()
470
471 pass
472 # -------------------------------------------------------------------------------------------------------------------- #
473
474
475 ########################################################################################################################
476 # Function to initialize an iRODS Session - will raise an Exception if timeout is longer than 2 seconds
477 #
478 # IN:
479 # String host
480 # String port
481 # String user
482 # String password
483 # String zone
484 #
485 # OUT:
486 # iRODSSession-object session
487 ########################################################################################################################
488 def get_iRODS_connection(host, port, user, password, zone):
489
490 # initialize timeout checker - fires after 2 secs
491 import signal
492 signal.signal(signal.SIGALRM, timeout_checker)
493 signal.alarm(2)
494
495 try:
496 session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone)
497 except Exception:
498 raise Exception("There was a timeout creating the iRODS session")
499
500 # void/reset alarm
501 signal.alarm(0)
502
503 return session
504 # -------------------------------------------------------------------------------------------------------------------- #
505
506
507 ########################################################################################################################
508 # Helper function to raise timeout exception when SIGALRM fires
509 #
510 # IN:
511 #
512 # OUT:
513 #
514 ########################################################################################################################
515 def timeout_checker():
516
517 raise Exception("iRODS session timeout")
518 # -------------------------------------------------------------------------------------------------------------------- #
519
520
521 ########################################################################################################################
522 # Function to check if file exists in iRODS
523 #
524 # IN:
525 # String path
526 #
527 # OUT:
528 # Bool ret_bool
529 #
530 ########################################################################################################################
531 def check_if_file_exists(path):
532
533 if os.path.isfile(path):
534 ret_bool = True
535 else:
536 ret_bool = False
537
538 return ret_bool
539 # -------------------------------------------------------------------------------------------------------------------- #
540
541
542 ########################################################################################################################
543 # Function to check iRODS destination
544 #
545 # IN:
546 # iRODSSession-object session
547 # String path
548 # String name
549 #
550 # OUT:
551 # Bool ret_bool
552 #
553 ########################################################################################################################
554 def check_iRODS_destination(session, path, name):
555
556 try:
557 session.collections.get(path.rstrip("/"))
558 except Exception:
559 raise Exception("Collection doesn't exist in iRODS file system")
560
561 try:
562 session.data_objects.get(path.rstrip("/") + "/" + name)
563 except Exception:
564 raise Exception("File doesn't exist in iRODS file system")
565 # -------------------------------------------------------------------------------------------------------------------- #
566
567
568 if __name__ == "__main__":
569 main()