Mercurial > repos > rhohensinner > galaxy_irods_interface

comparison main.py @ 4:84f685c067ad draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit b2a00d9c24285fef0fb131d1832ecf4c337e5038-dirty"
author rhohensinner
date Wed, 04 Aug 2021 12:01:55 +0000
parents d2be2eb8350f
children
comparison
equal deleted inserted replaced
3:d2be2eb8350f 4:84f685c067ad
1 ___author___ = "Richard Hohensinner"
2 ___created___ = "04.02.2021"
3 ___last_modified___ = "20.05.2021"
4
5 # general imports
6 import os, sys, json
7
8 from shutil import copyfile
9 from datetime import datetime
10
11 # irods-client imports
12 from irods.session import iRODSSession
13 from irods.models import Collection, DataObject
14 from irods.query import SpecificQuery
15
16 # Tkinter imports
17 from tkinter import messagebox, Label, Button, Entry, Listbox, Tk, PhotoImage, Grid
18
19 # global variables
20 session = None
21 file_path_list = []
22 selected_file = ""
23 session_success = False
24 selection_success = False
25 iRODSCredentials = {"host": "", "port": "", "user": "", "pw": "", "zone": ""}
26 python_path = []
27
28
29 ########################################################################################################################
30 # Main function of the iRODS tools
31 #
32 # IN:
33 # JSON Object params (argv[1])
34 # OUT:
35 #
36 ########################################################################################################################
37 def main():
38
39 # check input parameters
40 if len(sys.argv) == 2:
41 params = json.loads(sys.argv[1])
42 else:
43 raise Exception("Invalid Parameters submitted!")
44
45 tool_type = params["tool_type"]
46
47 try:
48 if params["tool_type"] != "up" and params["tool_type"] != "down":
49 raise Exception("Invalid tool-type parameter submitted!")
50 except:
51 raise Exception("No tool-type parameter submitted!")
52 is_download_call = True
53 if tool_type == "up":
54 is_download_call = False
55
56 # check params for integrity
57 result_string, params_faulty = check_params(params)
58
59 if params_faulty:
60 raise Exception(result_string)
61
62 global iRODSCredentials, session_success, selected_file, selection_success
63 iRODSCredentials["host"] = params["irods_host"]
64 iRODSCredentials["port"] = params["irods_port"]
65 iRODSCredentials["zone"] = params["irods_zone"]
66
67 # create login window
68 ## make_login_window(params)
69 iRODSCredentials["user"] = params["irods_user"]
70 iRODSCredentials["pw"] = params["irods_password"]
71
72 host = iRODSCredentials["host"]
73 port = iRODSCredentials["port"]
74 user = iRODSCredentials["user"]
75 password = iRODSCredentials["pw"]
76 zone = iRODSCredentials["zone"]
77
78 iRODSsession = get_iRODS_connection(host=host, port=port, user=user, password=password, zone=zone)
79 global session, session_success
80 try:
81 coll = iRODSsession.collections.get("/" + zone + "/" + "home" + "/" + user)
82 except Exception:
83 raise Exception("Invalid Login")
84
85 if coll:
86 session = iRODSsession
87 session_success = True
88
89
90 # check tool settings and start tool execution
91 if session_success:
92 # initialize download tool
93 if params["tool_type"] == "down":
94
95 selected_file = params["file_path"]
96 params["user"] = iRODSCredentials["pw"]
97 params["password"] = iRODSCredentials["user"]
98 # start download routine
99 handle_download_call(params)
100
101 # initialize upload tool
102 elif params["tool_type"] == "up":
103
104 if session_success:
105 params["user"] = iRODSCredentials["pw"]
106 params["password"] = iRODSCredentials["user"]
107 # start upload routine
108 handle_upload_call(params)
109 else:
110 raise Exception("Logging into iRODS failed")
111 else:
112 raise Exception("Logging into iRODS failed")
113 # -------------------------------------------------------------------------------------------------------------------- #
114
115
116 ########################################################################################################################
117 # Login Window class for Tkinter
118 #
119 # IN:
120 # Tk Window win
121 # OUT:
122 # (sets global variables iRODSCredentials, session and session_success)
123 #
124 ########################################################################################################################
125 class LoginWindow:
126 def __init__(self, win):
127 self.window = win
128 self.lbl1 = Label(win, text='iRODS Username:')
129 self.lbl2 = Label(win, text='iRODS Password:')
130 self.t1 = Entry(bd=3)
131 self.t2 = Entry(show="*")
132 self.b1 = Button(win, text='Login', command=self.login)
133
134 self.window.grid()
135 Grid.rowconfigure(self.window, 0, weight=1)
136 Grid.rowconfigure(self.window, 1, weight=1)
137 Grid.rowconfigure(self.window, 2, weight=1)
138 Grid.rowconfigure(self.window, 3, weight=1)
139 Grid.rowconfigure(self.window, 4, weight=1)
140 Grid.columnconfigure(self.window, 0, weight=1)
141
142 self.lbl1.grid(row=0, column=0, padx="20", pady="1", sticky="w")
143 self.t1.grid(row=1, column=0, padx="10", pady="1", sticky="nsew")
144 self.lbl2.grid(row=2, column=0, padx="20", pady="1", sticky="w")
145 self.t2.grid(row=3, column=0, padx="10", pady="1", sticky="nsew")
146 self.b1.grid(row=4, column=0, padx="50", pady="10", sticky="nsew")
147
148 def login(self):
149 global iRODSCredentials
150 user = str(self.t1.get())
151 password = str(self.t2.get())
152 if user == "" or password == "":
153 self.window.iconify()
154 messagebox.showerror("Error", "Username or Password empty!")
155 self.window.deiconify()
156 return
157 else:
158 iRODSCredentials["user"] = user
159 iRODSCredentials["pw"] = password
160
161 get_irods_session(self.window)
162 if not session_success:
163 return
164
165 self.window.destroy()
166 # -------------------------------------------------------------------------------------------------------------------- #
167
168
169 ########################################################################################################################
170 # File Selection Window class for Tkinter
171 #
172 # IN:
173 # Tk Window win
174 # OUT:
175 # (sets global variables selected_file and selection_success)
176 #
177 ########################################################################################################################
178 class FileSelectWindow:
179 def __init__(self, win):
180 global session, iRODSCredentials
181 self.session = session
182 self.window = win
183 self.b1 = Button(win, text='Select', command=self.select)
184 self.lb1 = Listbox(win)
185
186 self.window.grid()
187 Grid.rowconfigure(self.window, 0, weight=1)
188 Grid.rowconfigure(self.window, 1, weight=1)
189 Grid.columnconfigure(self.window, 0, weight=1)
190
191 self.lb1.grid(row=0, column=0, padx="20", pady="1", sticky="nswe")
192 self.b1.grid(row=1, column=0, padx="50", pady="1", sticky="ew")
193
194 coll = session.collections.get("/" + iRODSCredentials["zone"] + "/" + "home" + "/" + iRODSCredentials["user"])
195 file_list = []
196
197 self.get_files_from_collections(coll, file_list)
198
199 for counter in range(len(file_list)):
200 self.lb1.insert(counter, file_list[counter])
201
202 def get_files_from_collections(self, coll, file_list):
203 for obj in coll.data_objects:
204 file_list.append(obj.path)
205
206 for col in coll.subcollections:
207 self.get_files_from_collections(col, file_list)
208
209 def select(self):
210 global session, selected_file, selection_success
211 try:
212 selection = self.lb1.get(self.lb1.curselection())
213 except:
214 self.window.iconify()
215 messagebox.showerror("Error", "No file selected!")
216 self.window.deiconify()
217 return
218
219 selected_file = selection
220 selection_success = True
221 self.window.destroy()
222 # -------------------------------------------------------------------------------------------------------------------- #
223
224
225 ########################################################################################################################
226 # Creates an iRODS session and sets the global session variable
227 #
228 # IN:
229 # Tk Window window
230 #
231 # OUT:
232 #
233 ########################################################################################################################
234 def get_irods_session(window):
235 global iRODSCredentials
236 host = iRODSCredentials["host"]
237 port = iRODSCredentials["port"]
238 user = iRODSCredentials["user"]
239 password = iRODSCredentials["pw"]
240 zone = iRODSCredentials["zone"]
241
242 iRODSsession = get_iRODS_connection(host=host, port=port, user=user, password=password, zone=zone)
243 global session, session_success
244 try:
245 coll = iRODSsession.collections.get("/" + zone + "/" + "home" + "/" + user)
246 except Exception:
247 window.iconify()
248 messagebox.showerror("Error", "Invalid Authentification")
249 window.deiconify()
250 return
251
252 if coll:
253 session = iRODSsession
254 session_success = True
255 # -------------------------------------------------------------------------------------------------------------------- #
256
257
258 ########################################################################################################################
259 # Helper function to initialize Login Window classes and Tk windows
260 #
261 # IN:
262 #
263 # OUT:
264 #
265 ########################################################################################################################
266 def make_login_window(params):
267 #get login icon
268 log_img = ""
269 for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
270 for fn in filenames:
271 if fn == "irods_galaxy_login.png":
272 log_img = os.path.join(dirpath, fn)
273 if log_img != "":
274 break
275 if log_img != "":
276 break
277 # print(log_img)
278
279 window = Tk()
280 LoginWindow(window)
281 window.title('iRODS Login')
282 window.geometry("450x225+10+10")
283 window.minsize(450, 225)
284 window.tk.call('wm', 'iconphoto', window._w, PhotoImage(file=log_img))
285 # alternative options:
286 # window.iconphoto(False, PhotoImage(file='/path/to/ico/icon.png'))
287 # window.iconbitmap("/home/richard/git/galaxy_irods_tools/login.ico")
288 window.mainloop()
289 # -------------------------------------------------------------------------------------------------------------------- #
290
291
292 ########################################################################################################################
293 # Helper function to initialize File Selection Window classes and Tk windows
294 #
295 # IN:
296 #
297 # OUT:
298 #
299 ########################################################################################################################
300 def make_file_select_window():
301 window = Tk()
302 FileSelectWindow(window)
303 window.title('iRODS File Select')
304 window.geometry("450x225+10+10")
305 window.minsize(450, 225)
306 window.mainloop()
307 # -------------------------------------------------------------------------------------------------------------------- #
308
309
310 ########################################################################################################################
311 # Checks whether arguments are valid and returns true/false depending on params
312 #
313 # IN:
314 # Dict params
315 #
316 # OUT:
317 # String res_string
318 # Bool res_bool
319 ########################################################################################################################
320 def check_params(params):
321 res_string = ""
322 res_bool = False
323
324 try:
325 if params["irods_host"] == "":
326 res_string += "Host empty!\n"
327 res_bool = True
328 if params["irods_port"] == "":
329 res_string += "Port empty!\n"
330 res_bool = True
331 if params["irods_zone"] == "":
332 res_string += "Zone empty!\n"
333 res_bool = True
334 ##if params["selection_type"] == "path" and params["file_path"] == "":
335 ## res_string += "Missing file path!\n"
336 ## res_bool = True
337 except:
338 raise Exception("Invalid/Missing Parameters")
339
340 return res_string, res_bool
341 # -------------------------------------------------------------------------------------------------------------------- #
342
343
344 ########################################################################################################################
345 # Function to handle iRODS download calls
346 #
347 # IN:
348 # Dict params
349 #
350 # OUT:
351 #
352 ########################################################################################################################
353 def handle_download_call(params):
354
355 global session, selected_file
356
357 # check if /ZONE/USER/...FILE... pattern is valid
358 if len(selected_file.split("/")) < 2:
359 raise Exception("Path to file is not valid in iRODS")
360
361 file_list = []
362
363 # check if file is a directory
364 if "." not in selected_file:
365 try:
366 coll = session.collections.get(selected_file)
367 for file in coll.data_objects:
368 file_list.append(file.path)
369 except:
370 raise Exception("Invalid directory path specified!")
371 else:
372 file_list.append(selected_file)
373
374 # get registry file
375 reg_file = ""
376 for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
377 for fn in filenames:
378 if fn == "irods_galaxy_registry.xml":
379 reg_file = os.path.join(dirpath, fn)
380 if reg_file != "":
381 break
382 if reg_file != "":
383 break
384
385 # print(file_list)
386 # print(os.getcwd())
387
388 # handle download for all files in file_list
389 for f in file_list:
390
391 file_to_get = f
392
393 # handle path and file name
394 name_file_to_get = file_to_get.split("/")[-1]
395 path_file_to_get = "/".join(file_to_get.split("/")[0:len(file_to_get.split("/")) - 1])
396
397 # check iRODS filesystem
398 check_iRODS_destination(session, path_file_to_get, name_file_to_get)
399
400 # get file object from iRODS
401 iRODS_file_object = session.data_objects.get(path_file_to_get + "/" + name_file_to_get)
402 input_file = iRODS_file_object.open("r+")
403 output_file = open(name_file_to_get, "wb")
404 output_file.write(input_file.read())
405 #input_file.seek(0,0)
406 #print(input_file.read().decode("utf-8"))
407 #temp = open("/home/richard/test.txt", "w")
408 #input_file.seek(0,0)
409 #temp.write(input_file.read().decode("utf-8"))
410
411 input_file.close()
412 output_file.close()
413
414 abs_file_path = os.path.abspath(name_file_to_get)
415
416 file_type = str(name_file_to_get.split(".")[-1])
417
418 file_content = {"uuid": None,
419 "file_type": "auto",
420 "space_to_tab": False,
421 "dbkey": "?",
422 "to_posix_lines": True,
423 "ext": file_type,
424 "path": abs_file_path,
425 "in_place": True,
426 "dataset_id": params["job_id"],
427 "type": "file",
428 "is_binary": False,
429 "link_data_only": "copy_files",
430 "name": name_file_to_get
431 }
432
433 with open("temporal.json", "w") as fileParams:
434 fileParams.write(json.dumps(file_content))
435 fileParams.close()
436
437 # load file into Galaxy by using the integrated upload tool - Preparation
438 python_command = params["galaxy_root"] + "/tools/data_source/upload.py"
439 arg1 = params["galaxy_root"]
440 arg2 = params["galaxy_datatypes"]
441 arg3 = os.path.abspath(fileParams.name)
442 arg4 = params["job_id"] + ":" + params["out_dir"] + ":" + params["out_file"]
443 arg5 = params["galaxy_root"] + "/lib/galaxy"
444
445 # copy sample registry.xml to working directory
446 copyfile(reg_file, params["galaxy_datatypes"])
447
448 # activate environment for new process call and call the python upload command either both with
449 #sys.path.append(params["galaxy_root"] + "/lib")
450 #os.system("python -c \'import sys;sys.path.append(\"" + params["galaxy_root"] + "/lib\")\'" + "&& python3 " +
451 # python_command + " " + arg1 + " " + arg2 + " " + arg3 + " " + arg4)
452
453 # get upload file
454 upload_file = ""
455 for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
456 for fn in filenames:
457 if fn == "irods_upload.py":
458 upload_file = os.path.join(dirpath, fn)
459 if upload_file != "":
460 break
461 if upload_file != "":
462 break
463
464 global python_path
465 python_path = sys.path
466 os.system(params["galaxy_root"] + "/.venv/bin/python " + upload_file + " " + arg1 + " " + arg2 + " " + arg3 + " " + arg4)
467 # close connection
468 session.cleanup()
469 # -------------------------------------------------------------------------------------------------------------------- #
470
471
472 ########################################################################################################################
473 # Function to handle iRODS upload calls
474 #
475 # IN:
476 # Dict params
477 #
478 # OUT:
479 #
480 ########################################################################################################################
481 def handle_upload_call(params):
482
483 global session, iRODSCredentials
484
485 path_to_file = params["up_file_path"]
486 name_of_file = params["up_file"]
487 #print(path_to_file)
488 #print(name_of_file)
489
490 coll_path = "/" + iRODSCredentials["zone"] + "/home/" + iRODSCredentials["user"] + "/galaxyupload"
491 try:
492 coll = session.collections.get(coll_path)
493 except:
494 coll = session.collections.create(coll_path)
495
496 now = datetime.now()
497
498 # dd/mm/YY
499 day = now.strftime("%d%m%Y")
500 time = now.strftime("%H%M%S")
501
502 coll_path = coll_path + "/" + day
503
504 try:
505 coll = session.collections.get(coll_path)
506 except:
507 coll = session.collections.create(coll_path)
508
509 if "/" in name_of_file:
510 name_of_file = name_of_file.split("/")[-1]
511
512 irods_file_name = time + "_" + name_of_file
513 iRODS_file_object = session.data_objects.create(coll_path + "/" + irods_file_name)
514 iRODS_file_object = session.data_objects.get(coll_path + "/" + irods_file_name)
515
516 irods_file = iRODS_file_object.open("w")
517 galaxy_file = open(path_to_file, "rb")
518 content = galaxy_file.read()
519 irods_file.write(content)
520
521 print("Successfully uploaded: " + name_of_file + "\n as: " + irods_file_name)
522 # TODO can't close session without writing process finished - but reading/writing happens async.
523 # session.cleanup()
524
525 pass
526 # -------------------------------------------------------------------------------------------------------------------- #
527
528
529 ########################################################################################################################
530 # Function to initialize an iRODS Session - will raise an Exception if timeout is longer than 2 seconds
531 #
532 # IN:
533 # String host
534 # String port
535 # String user
536 # String password
537 # String zone
538 #
539 # OUT:
540 # iRODSSession-object session
541 ########################################################################################################################
542 def get_iRODS_connection(host, port, user, password, zone):
543
544 # initialize timeout checker - fires after 2 secs
545 import signal
546 signal.signal(signal.SIGALRM, timeout_checker)
547 signal.alarm(2)
548
549 try:
550 session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone)
551 except Exception:
552 raise Exception("There was a timeout creating the iRODS session")
553
554 # void/reset alarm
555 signal.alarm(0)
556
557 return session
558 # -------------------------------------------------------------------------------------------------------------------- #
559
560
561 ########################################################################################################################
562 # Helper function to raise timeout exception when SIGALRM fires
563 #
564 # IN:
565 #
566 # OUT:
567 #
568 ########################################################################################################################
569 def timeout_checker():
570
571 raise Exception("iRODS session timeout")
572 # -------------------------------------------------------------------------------------------------------------------- #
573
574
575 ########################################################################################################################
576 # Function to check if file exists in iRODS
577 #
578 # IN:
579 # String path
580 #
581 # OUT:
582 # Bool ret_bool
583 #
584 ########################################################################################################################
585 def check_if_file_exists(path):
586
587 if os.path.isfile(path):
588 ret_bool = True
589 else:
590 ret_bool = False
591
592 return ret_bool
593 # -------------------------------------------------------------------------------------------------------------------- #
594
595
596 ########################################################################################################################
597 # Function to check iRODS destination
598 #
599 # IN:
600 # iRODSSession-object session
601 # String path
602 # String name
603 #
604 # OUT:
605 # Bool ret_bool
606 #
607 ########################################################################################################################
608 def check_iRODS_destination(session, path, name):
609
610 try:
611 session.collections.get(path.rstrip("/"))
612 except Exception:
613 raise Exception("Collection doesn't exist in iRODS file system")
614
615 try:
616 session.data_objects.get(path.rstrip("/") + "/" + name)
617 except Exception:
618 raise Exception("File doesn't exist in iRODS file system")
619 # -------------------------------------------------------------------------------------------------------------------- #
620
621
622 if __name__ == "__main__":
623 main()