Mercurial > repos > rhohensinner > galaxy_irods_interface

comparison irods_main.py @ 4:84f685c067ad draft default tip

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
"planemo upload commit b2a00d9c24285fef0fb131d1832ecf4c337e5038-dirty"
author rhohensinner
date Wed, 04 Aug 2021 12:01:55 +0000
parents
children
comparison
equal deleted inserted replaced
3:d2be2eb8350f 4:84f685c067ad
1 ___author___ = "Richard Hohensinner"
2 ___created___ = "04.02.2021"
3 ___last_modified___ = "20.05.2021"
4
5 # general imports
6 import os, sys, json
7
8 from shutil import copyfile
9 from datetime import datetime
10 from irods_data import registry_content
11
12 # irods-client imports
13 from irods.session import iRODSSession
14 from irods.models import Collection, DataObject
15 from irods.query import SpecificQuery
16
17 # Tkinter imports
18 from tkinter import messagebox, Label, Button, Entry, Listbox, Tk, PhotoImage, Grid
19
20 # global variables
21 session = None
22 file_path_list = []
23 selected_file = ""
24 session_success = False
25 selection_success = False
26 iRODSCredentials = {"host": "", "port": "", "user": "", "pw": "", "zone": ""}
27 python_path = []
28
29 with open("reg.xml", "w") as xf:
30 xf.write(registry_content)
31 xf.close()
32
33 ########################################################################################################################
34 # Main function of the iRODS tools
35 #
36 # IN:
37 # JSON Object params (argv[1])
38 # OUT:
39 #
40 ########################################################################################################################
41 def main():
42
43 # check input parameters
44 if len(sys.argv) == 2:
45 params = json.loads(sys.argv[1])
46 else:
47 raise Exception("Invalid Parameters submitted!")
48
49 tool_type = params["tool_type"]
50
51 try:
52 if params["tool_type"] != "up" and params["tool_type"] != "down":
53 raise Exception("Invalid tool-type parameter submitted!")
54 except:
55 raise Exception("No tool-type parameter submitted!")
56 is_download_call = True
57 if tool_type == "up":
58 is_download_call = False
59
60 # check params for integrity
61 result_string, params_faulty = check_params(params)
62
63 if params_faulty:
64 raise Exception(result_string)
65
66 global iRODSCredentials, session_success, selected_file, selection_success
67 iRODSCredentials["host"] = params["irods_host"]
68 iRODSCredentials["port"] = params["irods_port"]
69 iRODSCredentials["zone"] = params["irods_zone"]
70
71 # create login window
72 ## make_login_window(params)
73 iRODSCredentials["user"] = params["irods_user"]
74 iRODSCredentials["pw"] = params["irods_password"]
75
76 host = iRODSCredentials["host"]
77 port = iRODSCredentials["port"]
78 user = iRODSCredentials["user"]
79 password = iRODSCredentials["pw"]
80 zone = iRODSCredentials["zone"]
81
82 iRODSsession = get_iRODS_connection(host=host, port=port, user=user, password=password, zone=zone)
83 global session, session_success
84 try:
85 coll = iRODSsession.collections.get("/" + zone + "/" + "home" + "/" + user)
86 except Exception:
87 raise Exception("Invalid Login")
88
89 if coll:
90 session = iRODSsession
91 session_success = True
92
93
94 # check tool settings and start tool execution
95 if session_success:
96 # initialize download tool
97 if params["tool_type"] == "down":
98
99 selected_file = params["file_path"]
100 params["user"] = iRODSCredentials["pw"]
101 params["password"] = iRODSCredentials["user"]
102 # start download routine
103 handle_download_call(params)
104
105 # initialize upload tool
106 elif params["tool_type"] == "up":
107
108 if session_success:
109 params["user"] = iRODSCredentials["pw"]
110 params["password"] = iRODSCredentials["user"]
111 # start upload routine
112 handle_upload_call(params)
113 else:
114 raise Exception("Logging into iRODS failed")
115 else:
116 raise Exception("Logging into iRODS failed")
117 # -------------------------------------------------------------------------------------------------------------------- #
118
119
120 ########################################################################################################################
121 # Login Window class for Tkinter
122 #
123 # IN:
124 # Tk Window win
125 # OUT:
126 # (sets global variables iRODSCredentials, session and session_success)
127 #
128 ########################################################################################################################
129 class LoginWindow:
130 def __init__(self, win):
131 self.window = win
132 self.lbl1 = Label(win, text='iRODS Username:')
133 self.lbl2 = Label(win, text='iRODS Password:')
134 self.t1 = Entry(bd=3)
135 self.t2 = Entry(show="*")
136 self.b1 = Button(win, text='Login', command=self.login)
137
138 self.window.grid()
139 Grid.rowconfigure(self.window, 0, weight=1)
140 Grid.rowconfigure(self.window, 1, weight=1)
141 Grid.rowconfigure(self.window, 2, weight=1)
142 Grid.rowconfigure(self.window, 3, weight=1)
143 Grid.rowconfigure(self.window, 4, weight=1)
144 Grid.columnconfigure(self.window, 0, weight=1)
145
146 self.lbl1.grid(row=0, column=0, padx="20", pady="1", sticky="w")
147 self.t1.grid(row=1, column=0, padx="10", pady="1", sticky="nsew")
148 self.lbl2.grid(row=2, column=0, padx="20", pady="1", sticky="w")
149 self.t2.grid(row=3, column=0, padx="10", pady="1", sticky="nsew")
150 self.b1.grid(row=4, column=0, padx="50", pady="10", sticky="nsew")
151
152 def login(self):
153 global iRODSCredentials
154 user = str(self.t1.get())
155 password = str(self.t2.get())
156 if user == "" or password == "":
157 self.window.iconify()
158 messagebox.showerror("Error", "Username or Password empty!")
159 self.window.deiconify()
160 return
161 else:
162 iRODSCredentials["user"] = user
163 iRODSCredentials["pw"] = password
164
165 get_irods_session(self.window)
166 if not session_success:
167 return
168
169 self.window.destroy()
170 # -------------------------------------------------------------------------------------------------------------------- #
171
172
173 ########################################################################################################################
174 # File Selection Window class for Tkinter
175 #
176 # IN:
177 # Tk Window win
178 # OUT:
179 # (sets global variables selected_file and selection_success)
180 #
181 ########################################################################################################################
182 class FileSelectWindow:
183 def __init__(self, win):
184 global session, iRODSCredentials
185 self.session = session
186 self.window = win
187 self.b1 = Button(win, text='Select', command=self.select)
188 self.lb1 = Listbox(win)
189
190 self.window.grid()
191 Grid.rowconfigure(self.window, 0, weight=1)
192 Grid.rowconfigure(self.window, 1, weight=1)
193 Grid.columnconfigure(self.window, 0, weight=1)
194
195 self.lb1.grid(row=0, column=0, padx="20", pady="1", sticky="nswe")
196 self.b1.grid(row=1, column=0, padx="50", pady="1", sticky="ew")
197
198 coll = session.collections.get("/" + iRODSCredentials["zone"] + "/" + "home" + "/" + iRODSCredentials["user"])
199 file_list = []
200
201 self.get_files_from_collections(coll, file_list)
202
203 for counter in range(len(file_list)):
204 self.lb1.insert(counter, file_list[counter])
205
206 def get_files_from_collections(self, coll, file_list):
207 for obj in coll.data_objects:
208 file_list.append(obj.path)
209
210 for col in coll.subcollections:
211 self.get_files_from_collections(col, file_list)
212
213 def select(self):
214 global session, selected_file, selection_success
215 try:
216 selection = self.lb1.get(self.lb1.curselection())
217 except:
218 self.window.iconify()
219 messagebox.showerror("Error", "No file selected!")
220 self.window.deiconify()
221 return
222
223 selected_file = selection
224 selection_success = True
225 self.window.destroy()
226 # -------------------------------------------------------------------------------------------------------------------- #
227
228
229 ########################################################################################################################
230 # Creates an iRODS session and sets the global session variable
231 #
232 # IN:
233 # Tk Window window
234 #
235 # OUT:
236 #
237 ########################################################################################################################
238 def get_irods_session(window):
239 global iRODSCredentials
240 host = iRODSCredentials["host"]
241 port = iRODSCredentials["port"]
242 user = iRODSCredentials["user"]
243 password = iRODSCredentials["pw"]
244 zone = iRODSCredentials["zone"]
245
246 iRODSsession = get_iRODS_connection(host=host, port=port, user=user, password=password, zone=zone)
247 global session, session_success
248 try:
249 coll = iRODSsession.collections.get("/" + zone + "/" + "home" + "/" + user)
250 except Exception:
251 window.iconify()
252 messagebox.showerror("Error", "Invalid Authentification")
253 window.deiconify()
254 return
255
256 if coll:
257 session = iRODSsession
258 session_success = True
259 # -------------------------------------------------------------------------------------------------------------------- #
260
261
262 ########################################################################################################################
263 # Helper function to initialize Login Window classes and Tk windows
264 #
265 # IN:
266 #
267 # OUT:
268 #
269 ########################################################################################################################
270 def make_login_window(params):
271 #get login icon
272 log_img = ""
273 for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
274 for fn in filenames:
275 if fn == "irods_galaxy_login.png":
276 log_img = os.path.join(dirpath, fn)
277 if log_img != "":
278 break
279 if log_img != "":
280 break
281 # print(log_img)
282
283 window = Tk()
284 LoginWindow(window)
285 window.title('iRODS Login')
286 window.geometry("450x225+10+10")
287 window.minsize(450, 225)
288 window.tk.call('wm', 'iconphoto', window._w, PhotoImage(file=log_img))
289 # alternative options:
290 # window.iconphoto(False, PhotoImage(file='/path/to/ico/icon.png'))
291 # window.iconbitmap("/home/richard/git/galaxy_irods_tools/login.ico")
292 window.mainloop()
293 # -------------------------------------------------------------------------------------------------------------------- #
294
295
296 ########################################################################################################################
297 # Helper function to initialize File Selection Window classes and Tk windows
298 #
299 # IN:
300 #
301 # OUT:
302 #
303 ########################################################################################################################
304 def make_file_select_window():
305 window = Tk()
306 FileSelectWindow(window)
307 window.title('iRODS File Select')
308 window.geometry("450x225+10+10")
309 window.minsize(450, 225)
310 window.mainloop()
311 # -------------------------------------------------------------------------------------------------------------------- #
312
313
314 ########################################################################################################################
315 # Checks whether arguments are valid and returns true/false depending on params
316 #
317 # IN:
318 # Dict params
319 #
320 # OUT:
321 # String res_string
322 # Bool res_bool
323 ########################################################################################################################
324 def check_params(params):
325 res_string = ""
326 res_bool = False
327
328 try:
329 if params["irods_host"] == "":
330 res_string += "Host empty!\n"
331 res_bool = True
332 if params["irods_port"] == "":
333 res_string += "Port empty!\n"
334 res_bool = True
335 if params["irods_zone"] == "":
336 res_string += "Zone empty!\n"
337 res_bool = True
338 ##if params["selection_type"] == "path" and params["file_path"] == "":
339 ## res_string += "Missing file path!\n"
340 ## res_bool = True
341 except:
342 raise Exception("Invalid/Missing Parameters")
343
344 return res_string, res_bool
345 # -------------------------------------------------------------------------------------------------------------------- #
346
347
348 ########################################################################################################################
349 # Function to handle iRODS download calls
350 #
351 # IN:
352 # Dict params
353 #
354 # OUT:
355 #
356 ########################################################################################################################
357 def handle_download_call(params):
358
359 global session, selected_file
360
361 # check if /ZONE/USER/...FILE... pattern is valid
362 if len(selected_file.split("/")) < 2:
363 raise Exception("Path to file is not valid in iRODS")
364
365 file_list = []
366
367 # check if file is a directory
368 if "." not in selected_file:
369 try:
370 coll = session.collections.get(selected_file)
371 for file in coll.data_objects:
372 file_list.append(file.path)
373 except:
374 raise Exception("Invalid directory path specified!")
375 else:
376 file_list.append(selected_file)
377
378 ## get registry file
379 #reg_file = ""
380 #for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
381 # for fn in filenames:
382 # if fn == "irods_galaxy_registry.xml":
383 # reg_file = os.path.join(dirpath, fn)
384 # if reg_file != "":
385 # break
386 # if reg_file != "":
387 # break
388 reg_file = "reg.xml"
389
390 # print(file_list)
391 # print(os.getcwd())
392
393 # handle download for all files in file_list
394 for f in file_list:
395
396 file_to_get = f
397
398 # handle path and file name
399 name_file_to_get = file_to_get.split("/")[-1]
400 path_file_to_get = "/".join(file_to_get.split("/")[0:len(file_to_get.split("/")) - 1])
401
402 # check iRODS filesystem
403 check_iRODS_destination(session, path_file_to_get, name_file_to_get)
404
405 # get file object from iRODS
406 iRODS_file_object = session.data_objects.get(path_file_to_get + "/" + name_file_to_get)
407 input_file = iRODS_file_object.open("r+")
408 output_file = open(name_file_to_get, "wb")
409 output_file.write(input_file.read())
410 #input_file.seek(0,0)
411 #print(input_file.read().decode("utf-8"))
412 #temp = open("/home/richard/test.txt", "w")
413 #input_file.seek(0,0)
414 #temp.write(input_file.read().decode("utf-8"))
415
416 input_file.close()
417 output_file.close()
418
419 abs_file_path = os.path.abspath(name_file_to_get)
420
421 file_type = str(name_file_to_get.split(".")[-1])
422
423 file_content = {"uuid": None,
424 "file_type": "auto",
425 "space_to_tab": False,
426 "dbkey": "?",
427 "to_posix_lines": True,
428 "ext": file_type,
429 "path": abs_file_path,
430 "in_place": True,
431 "dataset_id": params["job_id"],
432 "type": "file",
433 "is_binary": False,
434 "link_data_only": "copy_files",
435 "name": name_file_to_get
436 }
437
438 with open("temporal.json", "w") as fileParams:
439 fileParams.write(json.dumps(file_content))
440 fileParams.close()
441
442 # load file into Galaxy by using the integrated upload tool - Preparation
443 python_command = params["galaxy_root"] + "/tools/data_source/upload.py"
444 arg1 = params["galaxy_root"]
445 arg2 = params["galaxy_datatypes"]
446 arg3 = os.path.abspath(fileParams.name)
447 arg4 = params["job_id"] + ":" + params["out_dir"] + ":" + params["out_file"]
448 arg5 = params["galaxy_root"] + "/lib/galaxy"
449
450 # copy sample registry.xml to working directory
451 copyfile(reg_file, params["galaxy_datatypes"])
452
453 # activate environment for new process call and call the python upload command either both with
454 #sys.path.append(params["galaxy_root"] + "/lib")
455 #os.system("python -c \'import sys;sys.path.append(\"" + params["galaxy_root"] + "/lib\")\'" + "&& python3 " +
456 # python_command + " " + arg1 + " " + arg2 + " " + arg3 + " " + arg4)
457
458 # get upload file
459 upload_file = ""
460 for dirpath, dirnames, filenames in os.walk(params["galaxy_root"]):
461 for fn in filenames:
462 if fn == "irods_upload.py":
463 upload_file = os.path.join(dirpath, fn)
464 if upload_file != "":
465 break
466 if upload_file != "":
467 break
468 sys.path.append(params["galaxy_root"] + "/lib")
469 #sys.path.append(params["galaxy_root"] + "/tools/data_source")
470 #sys.path.append(params["galaxy_root"] + "/.venv/lib/python3.8/site-packages")
471 with open("python__path.txt", "w") as pp:
472 pp.truncate(0)
473 for item in sys.path:
474 pp.write(item + ",")
475
476 os.system(params["galaxy_root"] + "/.venv/bin/python " + upload_file + " " + arg1 + " " + arg2 + " " + arg3 + " " + arg4)
477 # close connection
478 session.cleanup()
479 # -------------------------------------------------------------------------------------------------------------------- #
480
481
482 ########################################################################################################################
483 # Function to handle iRODS upload calls
484 #
485 # IN:
486 # Dict params
487 #
488 # OUT:
489 #
490 ########################################################################################################################
491 def handle_upload_call(params):
492
493 global session, iRODSCredentials
494
495 path_to_file = params["up_file_path"]
496 name_of_file = params["up_file"]
497 #print(path_to_file)
498 #print(name_of_file)
499
500 coll_path = "/" + iRODSCredentials["zone"] + "/home/" + iRODSCredentials["user"] + "/galaxyupload"
501 try:
502 coll = session.collections.get(coll_path)
503 except:
504 coll = session.collections.create(coll_path)
505
506 now = datetime.now()
507
508 # dd/mm/YY
509 day = now.strftime("%d%m%Y")
510 time = now.strftime("%H%M%S")
511
512 coll_path = coll_path + "/" + day
513
514 try:
515 coll = session.collections.get(coll_path)
516 except:
517 coll = session.collections.create(coll_path)
518
519 if "/" in name_of_file:
520 name_of_file = name_of_file.split("/")[-1]
521
522 irods_file_name = time + "_" + name_of_file
523 iRODS_file_object = session.data_objects.create(coll_path + "/" + irods_file_name)
524 iRODS_file_object = session.data_objects.get(coll_path + "/" + irods_file_name)
525
526 irods_file = iRODS_file_object.open("w")
527 galaxy_file = open(path_to_file, "rb")
528 content = galaxy_file.read()
529 irods_file.write(content)
530
531 print("Successfully uploaded: " + name_of_file + "\n as: " + irods_file_name)
532 # TODO can't close session without writing process finished - but reading/writing happens async.
533 # session.cleanup()
534
535 pass
536 # -------------------------------------------------------------------------------------------------------------------- #
537
538
539 ########################################################################################################################
540 # Function to initialize an iRODS Session - will raise an Exception if timeout is longer than 2 seconds
541 #
542 # IN:
543 # String host
544 # String port
545 # String user
546 # String password
547 # String zone
548 #
549 # OUT:
550 # iRODSSession-object session
551 ########################################################################################################################
552 def get_iRODS_connection(host, port, user, password, zone):
553
554 # initialize timeout checker - fires after 2 secs
555 import signal
556 signal.signal(signal.SIGALRM, timeout_checker)
557 signal.alarm(2)
558
559 try:
560 session = iRODSSession(host=host, port=port, user=user, password=password, zone=zone)
561 except Exception:
562 raise Exception("There was a timeout creating the iRODS session")
563
564 # void/reset alarm
565 signal.alarm(0)
566
567 return session
568 # -------------------------------------------------------------------------------------------------------------------- #
569
570
571 ########################################################################################################################
572 # Helper function to raise timeout exception when SIGALRM fires
573 #
574 # IN:
575 #
576 # OUT:
577 #
578 ########################################################################################################################
579 def timeout_checker():
580
581 raise Exception("iRODS session timeout")
582 # -------------------------------------------------------------------------------------------------------------------- #
583
584
585 ########################################################################################################################
586 # Function to check if file exists in iRODS
587 #
588 # IN:
589 # String path
590 #
591 # OUT:
592 # Bool ret_bool
593 #
594 ########################################################################################################################
595 def check_if_file_exists(path):
596
597 if os.path.isfile(path):
598 ret_bool = True
599 else:
600 ret_bool = False
601
602 return ret_bool
603 # -------------------------------------------------------------------------------------------------------------------- #
604
605
606 ########################################################################################################################
607 # Function to check iRODS destination
608 #
609 # IN:
610 # iRODSSession-object session
611 # String path
612 # String name
613 #
614 # OUT:
615 # Bool ret_bool
616 #
617 ########################################################################################################################
618 def check_iRODS_destination(session, path, name):
619
620 try:
621 session.collections.get(path.rstrip("/"))
622 except Exception:
623 raise Exception("Collection doesn't exist in iRODS file system")
624
625 try:
626 session.data_objects.get(path.rstrip("/") + "/" + name)
627 except Exception:
628 raise Exception("File doesn't exist in iRODS file system")
629 # -------------------------------------------------------------------------------------------------------------------- #
630
631
632 if __name__ == "__main__":
633 main()