Mercurial > repos > earlhaminst > export_to_cluster
annotate export_to_cluster.py @ 3:9838eed606ad draft default tip
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
author | earlhaminst |
---|---|
date | Wed, 09 May 2018 05:52:40 -0400 |
parents | b97775e9fe06 |
children |
rev | line source |
---|---|
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
1 #!/usr/bin/env python |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
2 from __future__ import print_function |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
3 |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
4 import optparse |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
5 import os |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
6 import os.path |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
7 import re |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
8 import shutil |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
9 import sys |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
10 |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
11 parser = optparse.OptionParser() |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
12 parser.add_option('-d', '--export_dir', help='Directory where to export the datasets') |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
13 parser.add_option('-p', '--dir_prefix', help='How the export dir should start') |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
14 (options, args) = parser.parse_args() |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
15 if not options.export_dir: |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
16 parser.error('Export directory cannot be empty') |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
17 if not options.dir_prefix: |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
18 parser.error('Directory prefix cannot be empty') |
1
a9d1991c90e3
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 7d8a044afcc2cdf4fa7a3c360d9df3990356589c-dirty
earlhaminst
parents:
0
diff
changeset
|
19 if len(args) < 3: |
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
20 parser.error('Require at least two arguments') |
1
a9d1991c90e3
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 7d8a044afcc2cdf4fa7a3c360d9df3990356589c-dirty
earlhaminst
parents:
0
diff
changeset
|
21 if len(args) % 3 != 0: |
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
22 parser.error('Require an even number of arguments') |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
23 |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
24 real_export_dir = os.path.realpath(options.export_dir) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
25 dir_prefix = options.dir_prefix.rstrip(os.sep) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
26 if not real_export_dir.startswith(dir_prefix): |
3
9838eed606ad
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents:
2
diff
changeset
|
27 raise Exception("'%s' must be a subdirectory of '%s'" % (options.export_dir, dir_prefix)) |
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
28 if not os.path.exists(real_export_dir): |
3
9838eed606ad
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents:
2
diff
changeset
|
29 raise Exception("'%s' directory does not exist or it is not accessible by the Galaxy user" % options.export_dir) |
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
30 if not os.path.isdir(real_export_dir): |
3
9838eed606ad
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit 68b62fd9a9d5586ccd539779873af17124b7dbb5
earlhaminst
parents:
2
diff
changeset
|
31 raise Exception("'%s' is not a directory" % options.export_dir) |
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
32 |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
33 dataset_paths = args[::3] |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
34 dataset_names = args[1::3] |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
35 dataset_exts = args[2::3] |
2
b97775e9fe06
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents:
1
diff
changeset
|
36 exit_code = 0 |
0
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
37 for dp, dn, de in zip(dataset_paths, dataset_names, dataset_exts): |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
38 """ |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
39 Copied from get_valid_filename from django |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
40 https://github.com/django/django/blob/master/django/utils/text.py |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
41 """ |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
42 dn_de = "%s.%s" % (dn, de) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
43 dn_de_safe = re.sub(r'(?u)[^-\w.]', '', dn_de.strip().replace(' ', '_')) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
44 dest = os.path.join(real_export_dir, dn_de_safe) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
45 try: |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
46 shutil.copy2(dp, dest) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
47 print("Dataset '%s' copied to '%s'" % (dn, dest)) |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
48 except Exception as e: |
0dbe46f58158
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit e80af91bced56efdb4fbf62ac03232655a22f25d-dirty
earlhaminst
parents:
diff
changeset
|
49 msg = "Error copying dataset '%s' to '%s', %s" % (dn, dest, e) |
2
b97775e9fe06
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents:
1
diff
changeset
|
50 print(msg, file=sys.stderr) |
b97775e9fe06
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents:
1
diff
changeset
|
51 exit_code = 1 |
b97775e9fe06
planemo upload for repository https://github.com/TGAC/earlham-galaxytools/tree/master/tools/export_to_cluster/ commit f5c5f3d6ce676937f5c673ec7fc0631a9f490dc2
earlhaminst
parents:
1
diff
changeset
|
52 sys.exit(exit_code) |