annotate export_to_cluster.py @ 3:9838eed606ad draft default tip

planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
author earlhaminst
date Wed, 09 May 2018 05:52:40 -0400
parents b97775e9fe06
children
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
1 #!/usr/bin/env python
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
2 from __future__ import print_function
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
3
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
4 import optparse
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
5 import os
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
6 import os.path
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
7 import re
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
8 import shutil
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
9 import sys
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
10
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
11 parser = optparse.OptionParser()
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
12 parser.add_option('-d', '--export_dir', help='Directory where to export the datasets')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
13 parser.add_option('-p', '--dir_prefix', help='How the export dir should start')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
14 (options, args) = parser.parse_args()
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
15 if not options.export_dir:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
16 parser.error('Export directory cannot be empty')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
17 if not options.dir_prefix:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
18 parser.error('Directory prefix cannot be empty')
1
a9d1991c90e3 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 7d8a044afcc2cdf4fa7a3c360d9df3990356589c-dirty
earlhaminst
parents: 0
diff changeset
19 if len(args) < 3:
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
20 parser.error('Require at least two arguments')
1
a9d1991c90e3 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 7d8a044afcc2cdf4fa7a3c360d9df3990356589c-dirty
earlhaminst
parents: 0
diff changeset
21 if len(args) % 3 != 0:
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
22 parser.error('Require an even number of arguments')
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
23
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
24 real_export_dir = os.path.realpath(options.export_dir)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
25 dir_prefix = options.dir_prefix.rstrip(os.sep)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
26 if not real_export_dir.startswith(dir_prefix):
3
9838eed606ad planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents: 2
diff changeset
27 raise Exception("'%s' must be a subdirectory of '%s'" % (options.export_dir, dir_prefix))
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
28 if not os.path.exists(real_export_dir):
3
9838eed606ad planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents: 2
diff changeset
29 raise Exception("'%s' directory does not exist or it is not accessible by the Galaxy user" % options.export_dir)
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
30 if not os.path.isdir(real_export_dir):
3
9838eed606ad planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents: 2
diff changeset
31 raise Exception("'%s' is not a directory" % options.export_dir)
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
32
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
33 dataset_paths = args[::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
34 dataset_names = args[1::3]
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
35 dataset_exts = args[2::3]
2
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
36 exit_code = 0
0
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
37 for dp, dn, de in zip(dataset_paths, dataset_names, dataset_exts):
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
38 """
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
39 Copied from get_valid_filename from django
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
40 https://github.com/django/django/blob/master/django/utils/text.py
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
41 """
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
42 dn_de = "%s.%s" % (dn, de)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
43 dn_de_safe = re.sub(r'(?u)[^-\w.]', '', dn_de.strip().replace(' ', '_'))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
44 dest = os.path.join(real_export_dir, dn_de_safe)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
45 try:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
46 shutil.copy2(dp, dest)
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
47 print("Dataset '%s' copied to '%s'" % (dn, dest))
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
48 except Exception as e:
0dbe46f58158 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff changeset
49 msg = "Error copying dataset '%s' to '%s', %s" % (dn, dest, e)
2
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
50 print(msg, file=sys.stderr)
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
51 exit_code = 1
b97775e9fe06 planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents: 1
diff changeset
52 sys.exit(exit_code)